S3 transfer manager: single-file upload and download

Implements single-file Upload and Download for the SEP-compliant S3 Transfer
Manager (aws-cpp-sdk-s3-transfer), backed by aws-c-s3 meta requests via the
aws-crt-cpp S3 bindings. Covers file and stream sources/sinks, progress and
finish callbacks, checksum config, cancellation, and CRT-to-SEP error mapping.

Author: narahavi

cmake/sdksCommon.cmake

@@ -108,6 +108,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "redshift:tests/aws-cpp-sdk-redshift-integrati
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-unit-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-crt:tests/aws-cpp-sdk-s3-crt-integration-tests")
+list(APPEND SDK_TEST_PROJECT_LIST "s3-transfer:tests/aws-cpp-sdk-s3-transfer-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption-tests,tests/aws-cpp-sdk-s3-encryption-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests")

crt/aws-crt-cpp

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0.
 
 CRT_URI_PREFIX=https://codeload.github.com/awslabs
-CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/0463563f9f656a493ec22ca962c2464bc8b831ab  # v0.43.2
+CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/851d8d003c9d5150edab56807e2393013f3771de  # v0.43.4
 
 AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/4b5d524bf1a511b05e0fffe5bdc51800770b9427  # v0.10.4
 AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/8aa2a48a09f93c65d4cf06388e143a6584de6321  # v0.9.15
@@ -14,7 +14,7 @@ AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/8aefd899fc3210bfd0e3fd414011a3cb
 AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/e2946c99521fa12d285c9a0829c92b1bf713922b  # v0.27.5
 AWS_C_MQTT_URI=${CRT_URI_PREFIX}/aws-c-mqtt/zip/2ef9605ec9c50bea3f921e08022ddd57eed70901  # v0.16.0
 AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/a852faa2df3ab2b31fb4cfd64fd3379a2f4ae22e  # v0.13.2
-AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/528b9dfff4a804b334875ecf8a0471f7d1366f24  # v0.2.8
+AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/a1cc19f53b63658f1b1400b36f199eafeeb895a6  # v0.2.9
 AWS_CHECKSUMS_URI=${CRT_URI_PREFIX}/aws-checksums/zip/1d5f2f1f3e5d013aae8810878ceb5b3f6f258c4e  # v0.2.10
 AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/f6acf748df0ea6157d55e640730b38d21a7751cd  # v5.4.0
 S2N_URI=${CRT_URI_PREFIX}/s2n/zip/66b1c94d1dfc99b237427cbde230eca63bb8b89c  # v1.7.6

prefetch_crt_dependency.sh

@@ -0,0 +1,26 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+#include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/S3DownloadBuffer.h>
+
+namespace Aws {
+namespace S3 {
+namespace Transfer {
+
+/**
+ * Callback interface for zero-copy downloads. The transfer manager delivers each part of the
+ * object to OnDataReceived as it arrives, in object order. The buffer is move-only; move it out
+ * (e.g. auto held = std::move(buffer)) to retain the bytes past this call.
+ */
+class AWS_S3_TRANSFER_API DownloadDataReceiver {
+ public:
+  virtual ~DownloadDataReceiver() = default;
+  virtual void OnDataReceived(S3DownloadBuffer buffer) = 0;
+};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h

@@ -1,42 +1,41 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <future>
 #include <memory>
 #include <aws/s3-transfer/DownloadResponse.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Returned from S3TransferManager::Download to represent a single in-flight download. The
- * handle is move-only and owns the underlying transfer state.
- */
+class DownloadHandleImpl;
+
+// Move-only handle for a single in-flight download.
 class AWS_S3_TRANSFER_API DownloadHandle final {
 public:
-  DownloadHandle();
+  explicit DownloadHandle(Aws::UniquePtr<DownloadHandleImpl> impl);
   ~DownloadHandle();
+  DownloadHandle(const DownloadHandle&) = delete;
+  DownloadHandle& operator=(const DownloadHandle&) = delete;
   DownloadHandle(DownloadHandle&&) noexcept;
   DownloadHandle& operator=(DownloadHandle&&) noexcept;
 
-  /**
-   * Returns a future that resolves once the transfer finishes, succeeds, or fails.
-   */
+  // Resolves once the transfer finishes, succeeds, or fails.
   std::future<DownloadOutcome> CompletionFuture();
 
-  /**
-   * Requests cancellation of the in-flight download. Returns immediately; the future
-   * returned by CompletionFuture will resolve with a failure once the cancel takes effect.
-   */
+  // Returns immediately; the completion future resolves with a failure once the cancel takes effect.
   void Cancel();
-};
 
+private:
+  Aws::UniquePtr<DownloadHandleImpl> m_impl;
+};
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h

@@ -4,50 +4,88 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
-#include <aws/s3-transfer/DownloadProgressListener.h>
+#include <aws/s3-transfer/ProgressListener.h>
+#include <aws/s3-transfer/DownloadDataReceiver.h>
+#include <aws/core/client/AWSError.h>
+#include <aws/crt/Optional.h>
+#include <aws/core/utils/DateTime.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
 #include <aws/core/utils/memory/stl/AWSVector.h>
-#include <aws/core/utils/stream/ResponseStream.h>
+#include <aws/s3/S3Errors.h>
+#include <aws/s3/model/ChecksumMode.h>
 #include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/RequestPayer.h>
 #include <memory>
-#include <utility>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
+struct DownloadTransferState;
+
+namespace Internal {
+class DownloadRequestImpl;
+
+using OptionalError = Aws::Crt::Optional<Aws::Client::AWSError<Aws::S3::S3Errors>>;
+}
+
 /**
- * Request type for S3TransferManager::Download. Carries the inner S3 GetObjectRequest along
- * with the local destination (file path or stream factory) and any request-level progress
- * listeners. The transfer manager parallelizes large objects via ranged GETs internally.
+ * Request type for S3TransferManager::Download. All state lives behind m_impl, whose concrete type
+ * is chosen by the constructor: a destination file path selects the file strategy, a data receiver
+ * the stream strategy. Move-only, because that impl is uniquely owned; pass with std::move.
  */
 class AWS_S3_TRANSFER_API DownloadRequest final {
  public:
-  explicit DownloadRequest(
-    Aws::S3::Model::GetObjectRequest s3Request,
-    Aws::String destinationFilePath,
-    Aws::IOStreamFactory responseStreamFactory,
-    Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
-  : m_s3Request(std::move(s3Request)),
-    m_destinationFilePath(std::move(destinationFilePath)),
-    m_responseStreamFactory(std::move(responseStreamFactory)),
-    m_transferListeners(std::move(transferListeners)) {}
-
-  inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
-  inline const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; }
-  inline const Aws::IOStreamFactory& GetResponseStreamFactory() const { return m_responseStreamFactory; }
-  inline const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const {
-    return m_transferListeners;
-  }
+  DownloadRequest(Aws::String bucket,
+                  Aws::String key,
+                  Aws::String destinationFilePath,
+                  Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {});
+
+  DownloadRequest(Aws::String bucket,
+                  Aws::String key,
+                  std::shared_ptr<DownloadDataReceiver> dataReceiver,
+                  Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {});
+
+  ~DownloadRequest();
 
+  const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const;
+  // Empty for a receiver-backed download; otherwise where the object lands once it completes.
+  const Aws::String& GetDestinationFilePath() const;
+  const Aws::String& GetTempFilePath() const;
+  // Null for a file-backed download; otherwise the sink the CRT delivers body chunks to.
+  const std::shared_ptr<DownloadDataReceiver>& GetDataReceiver() const;
+
+  DownloadRequest& SetChecksumMode(Aws::S3::Model::ChecksumMode v);
+  DownloadRequest& SetExpectedBucketOwner(Aws::String v);
+  DownloadRequest& SetIfMatch(Aws::String v);
+  DownloadRequest& SetIfModifiedSince(Aws::Utils::DateTime v);
+  DownloadRequest& SetIfNoneMatch(Aws::String v);
+  DownloadRequest& SetIfUnmodifiedSince(Aws::Utils::DateTime v);
+  DownloadRequest& SetRange(Aws::String v);
+  DownloadRequest& SetRequestPayer(Aws::S3::Model::RequestPayer v);
+  DownloadRequest& SetResponseCacheControl(Aws::String v);
+  DownloadRequest& SetResponseContentDisposition(Aws::String v);
+  DownloadRequest& SetResponseContentEncoding(Aws::String v);
+  DownloadRequest& SetResponseContentLanguage(Aws::String v);
+  DownloadRequest& SetResponseContentType(Aws::String v);
+  DownloadRequest& SetResponseExpires(Aws::Utils::DateTime v);
+  DownloadRequest& SetSSECustomerAlgorithm(Aws::String v);
+  DownloadRequest& SetSSECustomerKey(Aws::String v);
+  DownloadRequest& SetSSECustomerKeyMD5(Aws::String v);
+  DownloadRequest& SetVersionId(Aws::String v);
+
+  const Aws::S3::Model::GetObjectRequest& GetS3Request() const;
+
+  Internal::OptionalError Validate() const;
+  Internal::OptionalError FinalizeOnSuccess(
+      const std::shared_ptr<DownloadTransferState>& state) const;
+  Internal::OptionalError CleanupOnFailure(const std::shared_ptr<DownloadTransferState>& state) const;
 
  private:
-  Aws::S3::Model::GetObjectRequest m_s3Request;
-  Aws::String m_destinationFilePath;
-  Aws::IOStreamFactory m_responseStreamFactory;
-  Aws::Vector<std::shared_ptr<DownloadProgressListener>> m_transferListeners;
+  std::shared_ptr<Internal::DownloadRequestImpl> m_impl;
 };
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadProgressListener.h

@@ -15,32 +15,30 @@ namespace S3 {
 namespace Transfer {
 
 /**
- * Response type returned via the DownloadHandle's future once the transfer completes. Wraps
- * the underlying S3 GetObjectResult with whole-object content length and range, regardless
- * of how many ranged GETs were issued internally.
+ * Response type returned via the DownloadHandle's future once the transfer completes. Wraps the
+ * underlying S3 GetObjectResult with whole-object content length and range. The S3 result is set
+ * at construction time so a DownloadResponse is never in a half-populated state; access it via
+ * GetS3Result().
  */
 class AWS_S3_TRANSFER_API DownloadResponse final {
  public:
+  // Default constructor exists to satisfy Aws::Utils::Outcome<R, E>, which default-constructs
+  // its R on the error path. On the success path the S3 result is always provided via the
+  // constructor below; a default-constructed instance is unreachable through DownloadOutcome
+  // because Outcome::GetResult() is only meaningful when IsSuccess() is true.
+  DownloadResponse() = default;
+
+  explicit DownloadResponse(Aws::S3::Model::GetObjectResult s3Result)
+    : m_s3Result(std::move(s3Result)) {}
+
   inline const Aws::S3::Model::GetObjectResult& GetS3Result() const { return m_s3Result; }
-  inline bool S3ResultHasBeenSet() const { return m_s3ResultHasBeenSet; }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  void SetS3Result(GetObjectResultT&& getS3Result) {
-    m_s3ResultHasBeenSet = true;
-    m_s3Result = std::forward<GetObjectResultT>(getS3Result);
-  }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  DownloadResponse& WithS3Result(GetObjectResultT&& getS3Result) {
-    SetS3Result(std::forward<GetObjectResultT>(getS3Result));
-    return *this;
-  }
 
  private:
   Aws::S3::Model::GetObjectResult m_s3Result;
-  bool m_s3ResultHasBeenSet = false;
 };
 
 using DownloadOutcome = Aws::Utils::Outcome<DownloadResponse, Aws::Client::AWSError<Aws::S3::S3Errors>>;
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadProgressSnapshot.h

@@ -4,16 +4,20 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/ProgressSnapshot.h>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
+class UploadRequest;
+class DownloadRequest;
+
 /**
  * Callback interface for receiving event-driven updates throughout the lifecycle of a transfer.
  * Subclass and override the events of interest; default implementations are empty so unused
  * callbacks can be ignored. Listeners may be registered on the request or on the manager.
- * Specialized via the UploadProgressListener and DownloadProgressListener type aliases.
+ * Specialized via the UploadProgressListener and DownloadProgressListener subclasses below.
  */
 template <typename RequestT, typename SnapshotT>
 class ProgressListener {
@@ -42,6 +46,22 @@ class ProgressListener {
   virtual void OnTransferFailed(const RequestT& /*request*/, const SnapshotT& /*snapshot*/) {}
 };
 
-}
-}
-}
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of an upload.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API UploadProgressListener
+    : public ProgressListener<UploadRequest, UploadProgressSnapshot> {};
+
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of a download.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API DownloadProgressListener
+    : public ProgressListener<DownloadRequest, DownloadProgressSnapshot> {};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws