Move S3 CRT related class out of s3-tranfer-manager module to s3 module (#3195)

* Move S3 CRT related class from s3-tranfer-manager module to s3 module

* Update AbstractS3AsyncClient

Author: zoewangg

services-custom/s3-transfer-manager/pom.xml

@@ -81,11 +81,6 @@
             <artifactId>aws-crt</artifactId>
             <version>${awscrt.version}</version>
         </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>http-client-spi</artifactId>
-            <version>${awsjavasdk.version}</version>
-        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>arns</artifactId>

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java

@@ -25,6 +25,7 @@
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.internal.resource.S3AccessPointResource;
 import software.amazon.awssdk.services.s3.internal.resource.S3ArnConverter;
 import software.amazon.awssdk.services.s3.internal.resource.S3Resource;

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java

@@ -45,6 +45,7 @@
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.async.DrainingSubscriber;
 import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerListenerTest.java

@@ -37,6 +37,7 @@
 import org.mockito.ArgumentCaptor;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerPauseAndResumeTest.java

@@ -30,6 +30,7 @@
 import software.amazon.awssdk.core.ResponseBytes;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
 import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java

@@ -75,6 +75,12 @@
             <artifactId>profiles</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk.crt</groupId>
+            <artifactId>aws-crt</artifactId>
+            <version>${awscrt.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <!-- Test Dependencies -->
         <dependency>
             <artifactId>commons-io</artifactId>

services/s3/pom.xml

@@ -13,7 +13,7 @@
  * permissions and limitations under the License.
  */
 
-package software.amazon.awssdk.transfer.s3;
+package software.amazon.awssdk.services.s3.crt;
 
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
@@ -24,12 +24,14 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.crt.CrtResource;
+import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
-import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.testutils.RandomTempFile;
-import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient;
+import software.amazon.awssdk.testutils.service.AwsTestBase;
 
 public class CrtExceptionTransformationIntegrationTest extends S3IntegrationTestBase {
 
@@ -39,30 +41,25 @@ public class CrtExceptionTransformationIntegrationTest extends S3IntegrationTest
 
     private static final int OBJ_SIZE = 8 * 1024;
     private static RandomTempFile testFile;
-    private static S3TransferManager transferManager;
     private static S3CrtAsyncClient s3Crt;
 
     @BeforeAll
-    public static void setupFixture() throws IOException {
-        createBucket(BUCKET);
+    public static void setupFixture() throws Exception {
+        S3IntegrationTestBase.setUp();
+        S3IntegrationTestBase.createBucket(BUCKET);
         testFile = new RandomTempFile(BUCKET, OBJ_SIZE);
         s3Crt = S3CrtAsyncClient.builder()
-                                .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
+                                .credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
                                 .region(S3IntegrationTestBase.DEFAULT_REGION)
                                 .build();
-        transferManager =
-            S3TransferManager.builder()
-                             .s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
-                                                          .region(S3IntegrationTestBase.DEFAULT_REGION))
-                             .build();
     }
 
     @AfterAll
     public static void tearDownFixture() {
-        deleteBucketAndAllContents(BUCKET);
+        S3IntegrationTestBase.deleteBucketAndAllContents(BUCKET);
         s3Crt.close();
-        transferManager.close();
         testFile.delete();
+        CrtResource.waitForNoResources();
     }
 
     @Test
@@ -83,28 +80,6 @@ public void getObjectNoSuchBucket() throws IOException {
                 .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist");
     }
 
-    @Test
-    public void transferManagerDownloadObjectWithNoSuchKey() throws IOException {
-        String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString();
-        assertThatThrownBy(() -> transferManager.downloadFile(DownloadFileRequest.builder()
-                                                                                 .getObjectRequest(GetObjectRequest.builder().bucket(BUCKET).key("randomKey").build())
-                                                                                 .destination(Paths.get(randomBaseDirectory).resolve("testFile"))
-                                                                                 .build()).completionFuture().join())
-                .hasCauseInstanceOf(NoSuchKeyException.class)
-                .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchKeyException: The specified key does not exist");
-    }
-
-    @Test
-    public void transferManagerDownloadObjectWithNoSuchBucket() throws IOException {
-        String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString();
-        assertThatThrownBy(() -> transferManager.downloadFile(DownloadFileRequest.builder()
-                                                                                 .getObjectRequest(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build())
-                                                                                 .destination(Paths.get(randomBaseDirectory).resolve("testFile"))
-                                                                                 .build()).completionFuture().join())
-                .hasCauseInstanceOf(NoSuchBucketException.class)
-                .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist");
-    }
-
     @Test
     public void putObjectNoSuchKey() throws IOException {
         String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString();
@@ -123,14 +98,4 @@ public void putObjectNoSuchBucket() throws IOException {
                 .hasCauseInstanceOf(NoSuchBucketException.class)
                 .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist");
     }
-
-    @Test
-    public void transferManagerUploadObjectWithNoSuchObject() throws IOException{
-        assertThatThrownBy(() -> transferManager.uploadFile(UploadFileRequest.builder()
-                                                                             .putObjectRequest(PutObjectRequest.builder().bucket("nonExistingTestBucket").key("someKey").build())
-                                                                             .source(testFile.toPath())
-                                                                             .build()).completionFuture().join())
-                .hasCauseInstanceOf(NoSuchBucketException.class)
-                .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist");
-    }
 }

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java renamed to services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/CrtExceptionTransformationIntegrationTest.java

@@ -13,7 +13,7 @@
  * permissions and limitations under the License.
  */
 
-package software.amazon.awssdk.transfer.s3;
+package software.amazon.awssdk.services.s3.crt;
 
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
 
@@ -35,10 +35,13 @@
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.crt.CrtResource;
+import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.utils.ChecksumUtils;
 import software.amazon.awssdk.testutils.RandomTempFile;
-import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient;
-import software.amazon.awssdk.transfer.s3.util.ChecksumUtils;
+import software.amazon.awssdk.testutils.service.AwsTestBase;
 
 public class S3CrtClientPutObjectIntegrationTest extends S3IntegrationTestBase {
     private static final String TEST_BUCKET = temporaryBucketName(S3CrtClientPutObjectIntegrationTest.class);
@@ -56,7 +59,7 @@ public static void setup() throws Exception {
         testFile = new RandomTempFile(TEST_KEY, OBJ_SIZE);
 
         s3Crt = S3CrtAsyncClient.builder()
-                                .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
+                                .credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN)
                                 .region(S3IntegrationTestBase.DEFAULT_REGION)
                                 .build();
     }
@@ -66,7 +69,7 @@ public static void teardown() throws IOException {
         S3IntegrationTestBase.deleteBucketAndAllContents(TEST_BUCKET);
         Files.delete(testFile.toPath());
         s3Crt.close();
-        S3IntegrationTestBase.cleanUp();
+        CrtResource.waitForNoResources();
     }
 
     @Test

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java renamed to services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrtClientPutObjectIntegrationTest.java

@@ -13,7 +13,7 @@
  * permissions and limitations under the License.
  */
 
-package software.amazon.awssdk.transfer.s3;
+package software.amazon.awssdk.services.s3.crt;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
@@ -31,12 +31,14 @@
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.async.SdkPublisher;
+import software.amazon.awssdk.crt.CrtResource;
 import software.amazon.awssdk.http.async.SimpleSubscriber;
+import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
+import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.testutils.RandomTempFile;
 import software.amazon.awssdk.testutils.service.AwsTestBase;
-import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient;
 import software.amazon.awssdk.utils.Md5Utils;
 
 public class S3CrtGetObjectIntegrationTest extends S3IntegrationTestBase {
@@ -47,7 +49,8 @@ public class S3CrtGetObjectIntegrationTest extends S3IntegrationTestBase {
     private static ExecutorService executorService;
 
     @BeforeAll
-    public static void setup() throws IOException {
+    public static void setup() throws Exception {
+        S3IntegrationTestBase.setUp();
         S3IntegrationTestBase.createBucket(BUCKET);
         crtClient = S3CrtAsyncClient.builder()
                                     .region(S3IntegrationTestBase.DEFAULT_REGION)
@@ -66,7 +69,7 @@ public static void cleanup() {
         crtClient.close();
         S3IntegrationTestBase.deleteBucketAndAllContents(BUCKET);
         executorService.shutdown();
-        S3IntegrationTestBase.cleanUp();
+        CrtResource.waitForNoResources();
     }
 
     @Test