Update TransferManager download APIs

services-custom/s3-transfermanager/pom.xml

@@ -66,7 +66,16 @@
             <artifactId>annotations</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>regions</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-core</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>service-test-utils</artifactId>

services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java

@@ -17,6 +17,8 @@
 
 import java.nio.file.Path;
 import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.utils.Validate;
 import software.amazon.awssdk.utils.builder.CopyableBuilder;
 import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
 
@@ -25,14 +27,17 @@
  */
 @SdkPublicApi
 public final class DownloadRequest implements TransferRequest, ToCopyableBuilder<DownloadRequest.Builder, DownloadRequest> {
-    private final String bucket;
-    private final String key;
     private final Path destination;
+    private final GetObjectRequest getObjectRequest;
 
     private DownloadRequest(BuilderImpl builder) {
-        this.bucket = builder.bucket;
-        this.key = builder.key;
+        Validate.isTrue((builder.bucket != null && builder.key != null) ^ builder.getObjectRequest != null,
+                        "Bucket key pair and the getObjectRequest can't both be configured");
         this.destination = builder.destination;
+        this.getObjectRequest = builder.getObjectRequest == null ? GetObjectRequest.builder()
+                                                                                   .bucket(builder.bucket)
+                                                                                   .key(builder.key)
+                                                                                   .build() : builder.getObjectRequest;
     }
 
     /**
@@ -49,13 +54,12 @@ public Builder toBuilder() {
 
     @Override
     public String bucket() {
-        return bucket;
+        return getObjectRequest.bucket();
     }
 
-
     @Override
     public String key() {
-        return key;
+        return getObjectRequest.key();
     }
 
     /**
@@ -68,7 +72,12 @@ public Path destination() {
         return destination;
     }
 
-    public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builder, DownloadRequest> {
+    public GetObjectRequest toApiRequest() {
+        return getObjectRequest;
+    }
+
+    public interface Builder extends TransferRequest.Builder<DownloadRequest, Builder>, CopyableBuilder<Builder,
+        DownloadRequest> {
 
         /**
          * The {@link Path} to file that response contents will be written to. The file must not exist or this method
@@ -79,6 +88,15 @@ public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builde
          */
         Builder destination(Path destination);
 
+
+        /**
+         * The {@link GetObjectRequest} request
+         *
+         * @param getObjectRequest the getObject request
+         * @return a reference to this object so that method calls can be chained together.
+         */
+        Builder apiRequest(GetObjectRequest getObjectRequest);
+
         /**
          * @return The built request.
          */
@@ -89,6 +107,7 @@ private static final class BuilderImpl implements Builder {
         private String bucket;
         private String key;
         private Path destination;
+        private GetObjectRequest getObjectRequest;
 
         private BuilderImpl() {
         }
@@ -111,6 +130,12 @@ public Builder key(String key) {
             return this;
         }
 
+        @Override
+        public Builder apiRequest(GetObjectRequest getObjectRequest) {
+            this.getObjectRequest = getObjectRequest;
+            return this;
+        }
+
         @Override
         public DownloadRequest build() {
             return new DownloadRequest(this);

services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/TransferRequest.java

@@ -37,21 +37,22 @@ public interface TransferRequest {
      */
     String key();
 
-
-    interface Builder {
+    interface Builder<TypeToBuildT, BuilderT extends Builder> {
 
         /**
          * The bucket name containing the object.
          *
          * @return Returns a reference to this object so that method calls can be chained together.
          */
-        Builder bucket(String bucket);
+        BuilderT bucket(String bucket);
 
         /**
          * The Key of the object to transfer.
          *
          * @return Returns a reference to this object so that method calls can be chained together.
          */
-        Builder key(String key);
+        BuilderT key(String key);
+
+        TypeToBuildT build();
     }
 }

services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java

@@ -64,7 +64,7 @@ public Builder toBuilder() {
     }
 
 
-    public interface Builder extends TransferRequest.Builder, CopyableBuilder<Builder, UploadRequest> {
+    public interface Builder extends TransferRequest.Builder<UploadRequest, Builder>, CopyableBuilder<Builder, UploadRequest> {
 
         /**
          * The {@link Path} to file containing data to send to the service. File will be read entirely and may be read

services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java

@@ -15,22 +15,31 @@
 
 package software.amazon.awssdk.custom.s3.transfer.internal;
 
+import java.util.ArrayList;
+import java.util.List;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.custom.s3.transfer.S3TransferManager;
 import software.amazon.awssdk.services.s3.S3CrtAsyncClient;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
 
 @SdkInternalApi
 public final class DefaultS3TransferManager implements S3TransferManager {
     private final S3CrtAsyncClient s3CrtAsyncClient;
+    private final List<SdkAutoCloseable> closables = new ArrayList<>();
 
     public DefaultS3TransferManager(DefaultBuilder builder) {
-        //TODO: create a managed S3CrtAsyncClient if it's not provided
-        this.s3CrtAsyncClient = builder.s3CrtAsyncClient;
+        if (builder.s3CrtAsyncClient == null) {
+            s3CrtAsyncClient = S3CrtAsyncClient.builder()
+                                               .build();
+            closables.add(s3CrtAsyncClient);
+        } else {
+            s3CrtAsyncClient = builder.s3CrtAsyncClient;
+        }
     }
 
     @Override
     public void close() {
-        s3CrtAsyncClient.close();
+        closables.forEach(SdkAutoCloseable::close);
     }
 
     public static Builder builder() {

services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.custom.s3.transfer.util;
+
+import org.junit.BeforeClass;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
+import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.testutils.service.AwsTestBase;
+
+/**
+ * Base class for S3 integration tests. Loads AWS credentials from a properties
+ * file and creates an S3 client for callers to use.
+ */
+public class S3IntegrationTestBase extends AwsTestBase {
+
+    protected static final Region DEFAULT_REGION = Region.US_WEST_2;
+    /**
+     * The S3 client for all tests to use.
+     */
+    protected static S3Client s3;
+
+    protected static S3AsyncClient s3Async;
+
+    /**
+     * Loads the AWS account info for the integration tests and creates an S3
+     * client for tests to use.
+     */
+    @BeforeClass
+    public static void setUp() throws Exception {
+        s3 = s3ClientBuilder().build();
+        s3Async = s3AsyncClientBuilder().build();
+    }
+
+    protected static S3ClientBuilder s3ClientBuilder() {
+        return S3Client.builder()
+                       .region(DEFAULT_REGION)
+                       .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
+    }
+
+    protected static S3AsyncClientBuilder s3AsyncClientBuilder() {
+        return S3AsyncClient.builder()
+                            .region(DEFAULT_REGION)
+                            .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
+    }
+
+    protected static void createBucket(String bucketName) {
+        createBucket(bucketName, 0);
+    }
+
+    private static void createBucket(String bucketName, int retryCount) {
+        try {
+            s3.createBucket(
+                CreateBucketRequest.builder()
+                                   .bucket(bucketName)
+                                   .createBucketConfiguration(
+                                       CreateBucketConfiguration.builder()
+                                                                .locationConstraint(BucketLocationConstraint.US_WEST_2)
+                                                                .build())
+                                   .build());
+        } catch (S3Exception e) {
+            System.err.println("Error attempting to create bucket: " + bucketName);
+            if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) {
+                System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName);
+            } else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) {
+                System.err.println("Printing all buckets for debug:");
+                s3.listBuckets().buckets().forEach(System.err::println);
+                if (retryCount < 2) {
+                    System.err.println("Retrying...");
+                    createBucket(bucketName, retryCount + 1);
+                } else {
+                    throw e;
+                }
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    protected static void deleteBucketAndAllContents(String bucketName) {
+        System.out.println("Deleting S3 bucket: " + bucketName);
+        ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
+
+        while (true) {
+            if (response.contents() == null) {
+                break;
+            }
+            for (S3Object objectSummary : response.contents()) {
+                s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
+            }
+
+            if (response.isTruncated()) {
+                response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
+            } else {
+                break;
+            }
+        }
+
+        ListObjectVersionsResponse versionsResponse = s3
+            .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
+        if (versionsResponse.versions() != null) {
+            for (ObjectVersion s : versionsResponse.versions()) {
+                s3.deleteObject(DeleteObjectRequest.builder()
+                                                   .bucket(bucketName)
+                                                   .key(s.key())
+                                                   .versionId(s.versionId())
+                                                   .build());
+            }
+        }
+
+        s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
+    }
+
+}

services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java

@@ -27,4 +27,7 @@
 @SdkPublicApi
 public interface S3CrtAsyncClientBuilder extends AwsClientBuilder<S3CrtAsyncClientBuilder, S3CrtAsyncClient> {
 
+    S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes);
+
+    S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps);
 }

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java

@@ -77,11 +77,13 @@ public S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride) {
         throw new UnsupportedOperationException();
     }
 
+    @Override
     public S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes) {
         this.partSizeBytes = partSizeBytes;
         return this;
     }
 
+    @Override
     public S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps) {
         this.maxThroughputGbps = maxThroughputGbps;
         return this;