add failure policy to download directory

Author: philasmar

.gitignore

@@ -69,4 +69,6 @@ sdk/test/Performance/**/BenchmarkDotNet.Artifacts/*
 #protocol-tests
 sdk/test/ProtocolTests/Generated/**/model
 sdk/test/ProtocolTests/Generated/**/sources
-sdk/test/ProtocolTests/Generated/**/build-info
+sdk/test/ProtocolTests/Generated/**/build-info
+
+.DS_Store

generator/.DevConfigs/c49077d9-90b3-437f-b316-6d8d8833ae76.json

@@ -0,0 +1,12 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "minor",
+      "changeLogMessages": [
+        "Add FailurePolicy property to TransferUtilityDownloadDirectoryRequest to allow configuration of failure handling behavior during directory downloads. The default behavior is set to abort on failure. Users can now choose to either abort the entire operation or continue downloading remaining files when a failure occurs.",
+        "Add ObjectDownloadFailedEvent event to TransferUtilityDownloadDirectory to notify users when an individual file download fails during a directory download operation. This event provides details about the failed download, including the original request, the specific file request and the exception encountered."
+      ]
+    }
+  ]
+}

sdk/src/Services/S3/Custom/Transfer/Internal/AbortOnFailurePolicy.cs

@@ -0,0 +1,72 @@
+/*******************************************************************************
+ *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *  this file except in compliance with the License. A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file.
+ *  This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *  CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *  specific language governing permissions and limitations under the License.
+ * *****************************************************************************
+ *    __  _    _  ___
+ *   (  )( \/\/ )/ __)
+ *   /__\ \    / \__ \
+ *  (_)(_) \/\/  (___/
+ *
+ *  AWS SDK for .NET
+ *  API Version: 2006-03-01
+ *
+ */
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Amazon.S3.Transfer.Internal
+{
+    /// <summary>
+    /// Failure policy that cancels all related operations and rethrows the exception when
+    /// an action fails.
+    /// </summary>
+    /// <remarks>
+    /// Use this policy when any single failure should abort the entire higher-level operation.
+    /// When an <see cref="Action"/> executed under this policy throws, the policy will cancel
+    /// the provided <see cref="CancellationTokenSource"/>, invoke an optional failure callback,
+    /// and then rethrow the exception so the caller can observe the original failure.
+    /// </remarks>
+    internal class AbortOnFailurePolicy : IFailurePolicy
+    {
+        /// <summary>
+        /// Executes the provided asynchronous <paramref name="action"/> under the abort-on-failure policy.
+        /// </summary>
+        /// <param name="action">An asynchronous delegate that performs the work to execute under the policy.</param>
+        /// <param name="onFailure">An optional callback that will be invoked with the exception if <paramref name="action"/> fails.</param>
+        /// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> that will be canceled by this policy to signal termination
+        /// of related work when a failure occurs.</param>
+        /// <returns>
+        /// A <see cref="Task{Boolean}"/> that completes with <c>true</c> when <paramref name="action"/> completes successfully.
+        /// If <paramref name="action"/> fails, this method cancels <paramref name="cancellationTokenSource"/>, invokes
+        /// <paramref name="onFailure"/> if provided, and rethrows the original exception; it does not return <c>false</c>.
+        /// </returns>
+        public async Task<bool> ExecuteAsync(Func<Task> action, Action<Exception> onFailure, CancellationTokenSource cancellationTokenSource)
+        {
+            try
+            {
+                await action().ConfigureAwait(false);
+
+                return true;
+            }
+            catch (Exception ex)
+            {
+                // Cancel all pending operations before propagating the exception
+                cancellationTokenSource.Cancel();
+                
+                onFailure?.Invoke(ex);
+                
+                throw;
+            }
+        }
+    }
+}

sdk/src/Services/S3/Custom/Transfer/Internal/ContinueOnFailurePolicy.cs

@@ -0,0 +1,104 @@
+/*******************************************************************************
+ *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *  this file except in compliance with the License. A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file.
+ *  This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *  CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *  specific language governing permissions and limitations under the License.
+ * *****************************************************************************
+ *    __  _    _  ___
+ *   (  )( \/\/ )/ __)
+ *   /__\ \    / \__ \
+ *  (_)(_) \/\/  (___/
+ *
+ *  AWS SDK for .NET
+ *  API Version: 2006-03-01
+ *
+ */
+
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Amazon.S3.Transfer.Internal
+{
+    /// <summary>
+    /// Failure policy that records exceptions and allows other operations to continue.
+    /// </summary>
+    /// <remarks>
+    /// Use this policy when individual operation failures should not abort the overall
+    /// download directory operation. Exceptions thrown by the action are captured and
+    /// stored in the supplied <see cref="ConcurrentBag{Exception}"/>, and an optional
+    /// onFailure callback is invoked. For cancellation triggered by
+    /// the provided <see cref="CancellationTokenSource"/>, cancellation is propagated
+    /// to callers by rethrowing the <see cref="OperationCanceledException"/>.
+    /// </remarks>
+    internal class ContinueOnFailurePolicy : IFailurePolicy
+    {
+        private readonly ConcurrentBag<Exception> _errors;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ContinueOnFailurePolicy"/> class.
+        /// </summary>
+        /// <param name="errors">A <see cref="ConcurrentBag{Exception}"/> used to collect exceptions
+        /// that occur while executing actions under this policy. Failures are added to this bag
+        /// so the caller can examine aggregated errors after the overall operation completes.</param>
+        internal ContinueOnFailurePolicy(ConcurrentBag<Exception> errors)
+        {
+            _errors = errors;
+        }
+
+        /// <summary>
+        /// Executes <paramref name="action"/> and records failures without throwing them,
+        /// unless the failure is an operation cancellation triggered by the provided
+        /// <paramref name="cancellationTokenSource"/>.
+        /// </summary>
+        /// <param name="action">The asynchronous operation to execute under the policy.</param>
+        /// <param name="onFailure">A callback invoked with the exception when <paramref name="action"/> fails.</param>
+        /// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> used to determine and signal cancellation.
+        /// The policy will rethrow cancellations when the cancellation token was requested.</param>
+        /// <returns>
+        /// A <see cref="Task{Boolean}"/> that completes with <c>true</c> when the action completed successfully.
+        /// If the action threw a non-cancellation exception, the exception is added to the internal error bag,
+        /// <paramref name="onFailure"/> is invoked if provided, and the method completes with <c>false</c> to indicate
+        /// the action failed but the policy handled it and allowed processing to continue.
+        /// </returns>
+        public async Task<bool> ExecuteAsync(Func<Task> action, Action<Exception> onFailure, CancellationTokenSource cancellationTokenSource)
+        {
+            try
+            {
+                await action().ConfigureAwait(false);
+
+                return true;
+            }
+            // If the operation was canceled via the provided token, propagate cancellation.
+            catch (OperationCanceledException ex) when (cancellationTokenSource?.IsCancellationRequested == true)
+            {
+                onFailure?.Invoke(ex);
+
+                // Collect the exception for later reporting.
+                _errors.Add(ex);
+                
+                throw;
+            }
+// Disabled warning CA1031 to allow catching all exceptions to continue processing.
+#pragma warning disable CA1031
+            catch (Exception ex)
+#pragma warning restore CA1031
+            {
+                onFailure?.Invoke(ex);
+
+                // Collect the exception for later reporting but don't throw it.
+                // This allows other downloads to continue processing.
+                _errors.Add(ex);
+
+                return false;
+            }
+        }
+    }
+}

sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs

@@ -20,6 +20,7 @@
  *
  */
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
@@ -30,11 +31,14 @@
 using Amazon.S3.Util;
 using Amazon.Util.Internal;
 using Amazon.Runtime;
+using Amazon.S3.Transfer.Model;
 
 namespace Amazon.S3.Transfer.Internal
 {
     internal partial class DownloadDirectoryCommand : BaseCommand<TransferUtilityDownloadDirectoryResponse>
     {
+        private IFailurePolicy _failurePolicy;
+        private ConcurrentBag<Exception> _errors;
         private readonly IAmazonS3 _s3Client;
         private readonly TransferUtilityDownloadDirectoryRequest _request;
         private readonly bool _skipEncryptionInstructionFiles;
@@ -52,6 +56,11 @@ internal DownloadDirectoryCommand(IAmazonS3 s3Client, TransferUtilityDownloadDir
             this._s3Client = s3Client;
             this._request = request;
             this._skipEncryptionInstructionFiles = s3Client is Amazon.S3.Internal.IAmazonS3Encryption;
+            _errors = request.FailurePolicy == FailurePolicy.ContinueOnFailure ? new ConcurrentBag<Exception>() : null;
+            _failurePolicy =
+                request.FailurePolicy == FailurePolicy.AbortOnFailure
+                    ? new AbortOnFailurePolicy()
+                    : new ContinueOnFailurePolicy(_errors);
         }
 
         private void downloadedProgressEventCallback(object sender, WriteObjectProgressArgs e)
@@ -107,12 +116,6 @@ internal TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(
             downloadRequest.IfNoneMatch = this._request.IfNoneMatch;
             downloadRequest.ResponseHeaderOverrides = this._request.ResponseHeaderOverrides;
 
-            //Ensure the target file is a rooted within LocalDirectory. Otherwise error.
-            if(!InternalSDKUtils.IsFilePathRootedWithDirectoryPath(downloadRequest.FilePath, _request.LocalDirectory))
-            {
-                throw new AmazonClientException($"The file {downloadRequest.FilePath} is not allowed outside of the target directory {_request.LocalDirectory}.");
-            }
-
             downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;
 
             return downloadRequest;

sdk/src/Services/S3/Custom/Transfer/Internal/IFailurePolicy.cs

@@ -0,0 +1,62 @@
+/*******************************************************************************
+ *  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *  Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *  this file except in compliance with the License. A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file.
+ *  This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *  CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *  specific language governing permissions and limitations under the License.
+ * *****************************************************************************
+ *    __  _    _  ___
+ *   (  )( \/\/ )/ __)
+ *   /__\ \    / \__ \
+ *  (_)(_) \/\/  (___/
+ *
+ *  AWS SDK for .NET
+ *  API Version: 2006-03-01
+ *
+ */
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Amazon.S3.Transfer.Internal
+{
+    /// <summary>
+    /// Defines a policy for handling failures when executing asynchronous operations.
+    /// Implementations encapsulate cancellation behavior for
+    /// operations that may fail and need controlled continuation or termination.
+    /// </summary>
+    internal interface IFailurePolicy
+    {
+        /// <summary>
+        /// Executes an asynchronous <paramref name="action"/> under this failure policy.
+        /// </summary>
+        /// <remarks>
+        /// Implementations of this interface control how failures that occur while running
+        /// <paramref name="action"/> are handled (for example, whether to abort the overall
+        /// operation, continue on failure, or aggregate errors). When <paramref name="action"/>
+        /// throws or faults, the policy implementation is responsible for invoking
+        /// <paramref name="onFailure"/> with the thrown <see cref="Exception"/> and for
+        /// taking any policy-specific cancellation action (for example by calling
+        /// <paramref name="cancellationTokenSource"/>.Cancel()).
+        ///
+        /// The returned <see cref="Task{Boolean}"/> completes with <c>true</c> when the
+        /// <paramref name="action"/> completed successfully according to the policy and
+        /// the caller may proceed. It completes with <c>false</c> when the action failed and
+        /// the policy handled the failure (the caller should treat this as a failed step).
+        /// </remarks>
+        /// <param name="action">A function that performs the asynchronous work to execute under the policy.</param>
+        /// <param name="onFailure">A callback that will be invoked with the exception when <paramref name="action"/> fails.</param>
+        /// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> the policy may cancel to signal termination of related work.</param>
+        /// <returns>
+        /// A <see cref="Task{Boolean}"/> that resolves to <c>true</c> if the action completed successfully
+        /// (no failure), or <c>false</c> if the action failed but the policy handled the failure.
+        /// </returns>
+        Task<bool> ExecuteAsync(Func<Task> action, Action<Exception> onFailure, CancellationTokenSource cancellationTokenSource);
+    }
+}

sdk/src/Services/S3/Custom/Transfer/Internal/_async/BaseCommand.async.cs

@@ -83,7 +83,7 @@ await completedTask
             }
         }
 
-        protected static async Task ExecuteCommandAsync<T>(BaseCommand<T> command, CancellationTokenSource internalCts, SemaphoreSlim throttler) where T : class
+        protected static async Task ExecuteCommandAsync<T>(BaseCommand<T> command, CancellationTokenSource internalCts) where T : class
         {
             try
             {
@@ -100,10 +100,6 @@ await command.ExecuteAsync(internalCts.Token)
                 }
                 throw;
             }
-            finally
-            {
-                throttler.Release();
-            }
         }
     }
 }