Update to use CRT DelegateCredentialsProvider

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java

@@ -15,7 +15,6 @@
 
 package software.amazon.awssdk.transfer.s3;
 
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
 

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialsProviderAdapter.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+
+import java.nio.charset.StandardCharsets;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.crt.auth.credentials.Credentials;
+import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
+import software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
+
+/**
+ * Adapts an SDK {@link AwsCredentialsProvider} to CRT {@link CredentialsProvider}
+ */
+@SdkInternalApi
+public final class CrtCredentialsProviderAdapter implements SdkAutoCloseable {
+    private final AwsCredentialsProvider credentialsProvider;
+
+    public CrtCredentialsProviderAdapter(AwsCredentialsProvider credentialsProvider) {
+        this.credentialsProvider = credentialsProvider;
+    }
+
+    public CredentialsProvider crtCredentials() {
+        return new DelegateCredentialsProvider.DelegateCredentialsProviderBuilder()
+            .withHandler(() -> {
+                AwsCredentials sdkCredentials = credentialsProvider.resolveCredentials();
+                byte[] accessKey = sdkCredentials.accessKeyId().getBytes(StandardCharsets.UTF_8);
+                byte[] secreteKey = sdkCredentials.secretAccessKey().getBytes(StandardCharsets.UTF_8);
+
+                // TODO: confirm with CRT if set empty means null. Currently setting null causes the crash
+                byte[] sessionTokens = new byte[0];
+                if (sdkCredentials instanceof AwsSessionCredentials) {
+                    sessionTokens =
+                        ((AwsSessionCredentials) sdkCredentials).sessionToken().getBytes(StandardCharsets.UTF_8);
+                }
+
+                return new Credentials(accessKey,
+                                       secreteKey,
+                                       sessionTokens);
+            }).build();
+    }
+
+    @Override
+    public void close() {
+        if (credentialsProvider instanceof SdkAutoCloseable) {
+            ((SdkAutoCloseable) credentialsProvider).close();
+        }
+    }
+}

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java

@@ -39,14 +39,13 @@ public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
         S3NativeClientConfiguration.Builder configBuilder =
             S3NativeClientConfiguration.builder()
                                        .targetThroughputInGbps(builder.targetThroughputInGbps())
-                                       .partSizeInBytes(builder.partSizeBytes());
+                                       .partSizeInBytes(builder.minimumPartSizeInBytes())
+                                       .maxConcurrency(builder.maxConcurrency)
+                                       .credentialsProvider(builder.credentialsProvider);
         if (builder.region() != null) {
             configBuilder.signingRegion(builder.region().id());
         }
 
-        if (builder.credentialsProvider() != null) {
-            configBuilder.credentialsProvider(S3CrtPojoConversion.createCrtCredentialsProvider(builder.credentialsProvider()));
-        }
         configuration = configBuilder.build();
 
         this.s3NativeClient = new S3NativeClient(configuration.signingRegion(),
@@ -114,7 +113,7 @@ private static RequestDataSupplier adaptToDataSupplier(AsyncRequestBody requestB
     public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder {
         private AwsCredentialsProvider credentialsProvider;
         private Region region;
-        private Long partSizeBytes;
+        private Long minimalPartSizeInBytes;
         private Double targetThroughputInGbps;
         private Integer maxConcurrency;
 
@@ -126,8 +125,8 @@ public Region region() {
             return region;
         }
 
-        public Long partSizeBytes() {
-            return partSizeBytes;
+        public Long minimumPartSizeInBytes() {
+            return minimalPartSizeInBytes;
         }
 
         public Double targetThroughputInGbps() {
@@ -152,7 +151,7 @@ public S3CrtAsyncClientBuilder region(Region region) {
 
         @Override
         public S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long partSizeBytes) {
-            this.partSizeBytes = partSizeBytes;
+            this.minimalPartSizeInBytes = partSizeBytes;
             return this;
         }
 

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java

@@ -23,21 +23,15 @@
 import com.amazonaws.s3.model.RequestPayer;
 import com.amazonaws.s3.model.ServerSideEncryption;
 import com.amazonaws.s3.model.StorageClass;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 import software.amazon.awssdk.annotations.SdkInternalApi;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
 import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.awscore.DefaultAwsResponseMetadata;
 import software.amazon.awssdk.core.util.SdkUserAgent;
-import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
-import software.amazon.awssdk.crt.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.crt.http.HttpHeader;
 import software.amazon.awssdk.http.SdkHttpResponse;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -59,24 +53,6 @@ public final class S3CrtPojoConversion {
     private S3CrtPojoConversion() {
     }
 
-    // TODO: Move this method out of this util class and change to use DelegateCredentialsProvider
-    /**
-     * Adapter between the sdk credentials provider and the crt credentials provider.
-     */
-    public static CredentialsProvider createCrtCredentialsProvider(AwsCredentialsProvider awsCredentialsProvider) {
-        AwsCredentials sdkCredentials = awsCredentialsProvider.resolveCredentials();
-        StaticCredentialsProvider.StaticCredentialsProviderBuilder builder =
-            new StaticCredentialsProvider.StaticCredentialsProviderBuilder();
-
-        if (sdkCredentials instanceof AwsSessionCredentials) {
-            builder.withSessionToken(((AwsSessionCredentials) sdkCredentials).sessionToken().getBytes(StandardCharsets.UTF_8));
-        }
-
-        return builder.withAccessKeyId(sdkCredentials.accessKeyId().getBytes(StandardCharsets.UTF_8))
-                      .withSecretAccessKey(sdkCredentials.secretAccessKey().getBytes(StandardCharsets.UTF_8))
-                      .build();
-    }
-
     public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetObjectRequest request) {
         com.amazonaws.s3.model.GetObjectRequest.Builder getObjectBuilder =
             com.amazonaws.s3.model.GetObjectRequest.builder()
@@ -110,7 +86,6 @@ public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetO
 
     }
 
-    // TODO: codegen and add tests
     public static GetObjectResponse fromCrtGetObjectOutput(GetObjectOutput response, SdkHttpResponse sdkHttpResponse) {
         S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse);
 
@@ -236,7 +211,6 @@ public static com.amazonaws.s3.model.PutObjectRequest toCrtPutObjectRequest(PutO
         return putObjectBuilder.build();
     }
 
-    // TODO: codegen and add tests
     public static PutObjectResponse fromCrtPutObjectOutput(PutObjectOutput crtPutObjectOutput) {
         // TODO: Provide the HTTP request-level data (e.g. response metadata, HTTP response)
         PutObjectResponse.Builder builder = PutObjectResponse.builder()

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java

@@ -17,8 +17,9 @@
 
 
 import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
 import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
-import software.amazon.awssdk.crt.auth.credentials.DefaultChainCredentialsProvider;
 import software.amazon.awssdk.crt.io.ClientBootstrap;
 import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
 import software.amazon.awssdk.transfer.s3.SizeConstant;
@@ -33,6 +34,7 @@ public final class S3NativeClientConfiguration implements SdkAutoCloseable {
     private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 5;
     private final String signingRegion;
     private final ClientBootstrap clientBootstrap;
+    private final CrtCredentialsProviderAdapter credentialProviderAdapter;
     private final CredentialsProvider credentialsProvider;
     private final long partSizeInBytes;
     private final double targetThroughputInGbps;
@@ -42,11 +44,12 @@ public S3NativeClientConfiguration(Builder builder) {
         this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
                              builder.signingRegion;
         this.clientBootstrap = new ClientBootstrap(null, null);
-        this.credentialsProvider = builder.credentialsProvider == null ?
-                                   new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder()
-                                       .withClientBootstrap(clientBootstrap)
-                                       .build() :
-                                   builder.credentialsProvider;
+        this.credentialProviderAdapter =
+            builder.credentialsProvider == null ?
+            new CrtCredentialsProviderAdapter(DefaultCredentialsProvider.create()) :
+            new CrtCredentialsProviderAdapter(builder.credentialsProvider);
+        this.credentialsProvider = credentialProviderAdapter.crtCredentials();
+
         this.partSizeInBytes = builder.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES :
                                builder.partSizeInBytes;
         this.targetThroughputInGbps = builder.targetThroughputInGbps == null ?
@@ -87,12 +90,13 @@ public int maxConcurrency() {
     @Override
     public void close() {
         clientBootstrap.close();
+        credentialProviderAdapter.close();
         credentialsProvider.close();
     }
 
     public static final class Builder {
         private String signingRegion;
-        private CredentialsProvider credentialsProvider;
+        private AwsCredentialsProvider credentialsProvider;
         private Long partSizeInBytes;
         private Double targetThroughputInGbps;
         private Integer maxConcurrency;
@@ -105,7 +109,7 @@ public Builder signingRegion(String signingRegion) {
             return this;
         }
 
-        public Builder credentialsProvider(CredentialsProvider credentialsProvider) {
+        public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
             this.credentialsProvider = credentialsProvider;
             return this;
         }

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialProviderAdapterTest.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import org.junit.Test;
+import org.mockito.Mockito;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.crt.auth.credentials.Credentials;
+import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
+
+public class CrtCredentialProviderAdapterTest {
+
+    @Test
+    public void crtCredentials_withSession_shouldConvert() {
+        AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
+            .create(AwsSessionCredentials.create("foo", "bar", "session"));
+
+        CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider)
+            .crtCredentials();
+
+        Credentials credentials = crtCredentialsProvider.getCredentials().join();
+
+        assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
+        assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
+        assertThat(credentials.getSessionToken()).isEqualTo("session".getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Test
+    public void crtCredentials_withoutSession_shouldConvert() {
+        AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
+            .create(AwsBasicCredentials.create("foo", "bar"));
+
+        CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider)
+            .crtCredentials();
+
+        Credentials credentials = crtCredentialsProvider.getCredentials().join();
+
+        assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
+        assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
+        assertThat(credentials.getSessionToken()).isNull();
+    }
+
+    @Test
+    public void crtCredentials_provideAwsCredentials_shouldInvokeResolveAndClose() {
+        HttpCredentialsProvider awsCredentialsProvider = Mockito.mock(HttpCredentialsProvider.class);
+        AwsCredentials credentials = new AwsCredentials() {
+            @Override
+            public String accessKeyId() {
+                return "foo";
+            }
+
+            @Override
+            public String secretAccessKey() {
+                return "bar";
+            }
+        };
+        when(awsCredentialsProvider.resolveCredentials()).thenReturn(credentials);
+
+        CrtCredentialsProviderAdapter adapter = new CrtCredentialsProviderAdapter(awsCredentialsProvider);
+        CredentialsProvider crtCredentialsProvider = adapter.crtCredentials();
+
+        Credentials crtCredentials = crtCredentialsProvider.getCredentials().join();
+        assertThat(crtCredentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
+        assertThat(crtCredentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
+        verify(awsCredentialsProvider).resolveCredentials();
+
+        adapter.close();
+        verify(awsCredentialsProvider).close();
+    }
+}

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java

@@ -23,28 +23,21 @@
 import com.amazonaws.s3.model.PutObjectOutput;
 import com.amazonaws.s3.model.ReplicationStatus;
 import java.lang.reflect.Field;
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.ExecutionException;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.Test;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
 import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
 import software.amazon.awssdk.core.ApiName;
 import software.amazon.awssdk.core.SdkField;
 import software.amazon.awssdk.core.util.SdkUserAgent;
-import software.amazon.awssdk.crt.auth.credentials.Credentials;
-import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
 import software.amazon.awssdk.crt.http.HttpHeader;
 import software.amazon.awssdk.http.SdkHttpResponse;
 import software.amazon.awssdk.metrics.LoggingMetricPublisher;
@@ -67,19 +60,6 @@ public class S3CrtPojoConversionTest {
     private static final String SECRET_ACCESS_KEY = "secretAccessKey";
     private static final String SESSION_TOKEN = "sessionToken";
 
-    @Test
-    public void createCrtCredentialsProviderTest() throws ExecutionException, InterruptedException {
-        AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
-            .create(AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, SESSION_TOKEN));
-        CredentialsProvider crtCredentialsProvider = S3CrtPojoConversion.createCrtCredentialsProvider(awsCredentialsProvider);
-
-        Credentials credentials = crtCredentialsProvider.getCredentials().get();
-
-        assertThat(ACCESS_KEY.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getAccessKeyId());
-        assertThat(SECRET_ACCESS_KEY.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getSecretAccessKey());
-        assertThat(SESSION_TOKEN.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getSessionToken());
-    }
-
     @Test
     public void fromCrtPutObjectOutputAllFields_shouldConvert() throws IllegalAccessException {