Skip to content

Commit c5ac019

Browse files
zoewanggzoewangg
committed
Update to use CRT DelegateCredentialsProvider
1 parent b78f5b3 commit c5ac019

File tree

7 files changed

+178
-63
lines changed

7 files changed

+178
-63
lines changed

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
package software.amazon.awssdk.transfer.s3;
1717

18-
1918
import static org.assertj.core.api.Assertions.assertThat;
2019
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
2120

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialsProviderAdapter.java

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3.internal;
17+
18+
19+
import java.nio.charset.StandardCharsets;
20+
import software.amazon.awssdk.annotations.SdkInternalApi;
21+
import software.amazon.awssdk.auth.credentials.AwsCredentials;
22+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
23+
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
24+
import software.amazon.awssdk.crt.auth.credentials.Credentials;
25+
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
26+
import software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider;
27+
import software.amazon.awssdk.utils.SdkAutoCloseable;
28+
29+
/**
30+
* Adapts an SDK {@link AwsCredentialsProvider} to CRT {@link CredentialsProvider}
31+
*/
32+
@SdkInternalApi
33+
public final class CrtCredentialsProviderAdapter implements SdkAutoCloseable {
34+
private final AwsCredentialsProvider credentialsProvider;
35+
36+
public CrtCredentialsProviderAdapter(AwsCredentialsProvider credentialsProvider) {
37+
this.credentialsProvider = credentialsProvider;
38+
}
39+
40+
public CredentialsProvider crtCredentials() {
41+
return new DelegateCredentialsProvider.DelegateCredentialsProviderBuilder()
42+
.withHandler(() -> {
43+
AwsCredentials sdkCredentials = credentialsProvider.resolveCredentials();
44+
byte[] accessKey = sdkCredentials.accessKeyId().getBytes(StandardCharsets.UTF_8);
45+
byte[] secreteKey = sdkCredentials.secretAccessKey().getBytes(StandardCharsets.UTF_8);
46+
47+
// TODO: confirm with CRT if set empty means null. Currently setting null causes the crash
48+
byte[] sessionTokens = new byte[0];
49+
if (sdkCredentials instanceof AwsSessionCredentials) {
50+
sessionTokens =
51+
((AwsSessionCredentials) sdkCredentials).sessionToken().getBytes(StandardCharsets.UTF_8);
52+
}
53+
54+
return new Credentials(accessKey,
55+
secreteKey,
56+
sessionTokens);
57+
}).build();
58+
}
59+
60+
@Override
61+
public void close() {
62+
if (credentialsProvider instanceof SdkAutoCloseable) {
63+
((SdkAutoCloseable) credentialsProvider).close();
64+
}
65+
}
66+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,13 @@ public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
3939
S3NativeClientConfiguration.Builder configBuilder =
4040
S3NativeClientConfiguration.builder()
4141
.targetThroughputInGbps(builder.targetThroughputInGbps())
42-
.partSizeInBytes(builder.partSizeBytes());
42+
.partSizeInBytes(builder.minimumPartSizeInBytes())
43+
.maxConcurrency(builder.maxConcurrency)
44+
.credentialsProvider(builder.credentialsProvider);
4345
if (builder.region() != null) {
4446
configBuilder.signingRegion(builder.region().id());
4547
}
4648

47-
if (builder.credentialsProvider() != null) {
48-
configBuilder.credentialsProvider(S3CrtPojoConversion.createCrtCredentialsProvider(builder.credentialsProvider()));
49-
}
5049
configuration = configBuilder.build();
5150

5251
this.s3NativeClient = new S3NativeClient(configuration.signingRegion(),
@@ -114,7 +113,7 @@ private static RequestDataSupplier adaptToDataSupplier(AsyncRequestBody requestB
114113
public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder {
115114
private AwsCredentialsProvider credentialsProvider;
116115
private Region region;
117-
private Long partSizeBytes;
116+
private Long minimalPartSizeInBytes;
118117
private Double targetThroughputInGbps;
119118
private Integer maxConcurrency;
120119

@@ -126,8 +125,8 @@ public Region region() {
126125
return region;
127126
}
128127

129-
public Long partSizeBytes() {
130-
return partSizeBytes;
128+
public Long minimumPartSizeInBytes() {
129+
return minimalPartSizeInBytes;
131130
}
132131

133132
public Double targetThroughputInGbps() {
@@ -152,7 +151,7 @@ public S3CrtAsyncClientBuilder region(Region region) {
152151

153152
@Override
154153
public S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long partSizeBytes) {
155-
this.partSizeBytes = partSizeBytes;
154+
this.minimalPartSizeInBytes = partSizeBytes;
156155
return this;
157156
}
158157

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,15 @@
2323
import com.amazonaws.s3.model.RequestPayer;
2424
import com.amazonaws.s3.model.ServerSideEncryption;
2525
import com.amazonaws.s3.model.StorageClass;
26-
import java.nio.charset.StandardCharsets;
2726
import java.util.ArrayList;
2827
import java.util.HashMap;
2928
import java.util.List;
3029
import java.util.Map;
3130
import java.util.function.Consumer;
3231
import software.amazon.awssdk.annotations.SdkInternalApi;
33-
import software.amazon.awssdk.auth.credentials.AwsCredentials;
34-
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
35-
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
3632
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
3733
import software.amazon.awssdk.awscore.DefaultAwsResponseMetadata;
3834
import software.amazon.awssdk.core.util.SdkUserAgent;
39-
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
40-
import software.amazon.awssdk.crt.auth.credentials.StaticCredentialsProvider;
4135
import software.amazon.awssdk.crt.http.HttpHeader;
4236
import software.amazon.awssdk.http.SdkHttpResponse;
4337
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -59,24 +53,6 @@ public final class S3CrtPojoConversion {
5953
private S3CrtPojoConversion() {
6054
}
6155

62-
// TODO: Move this method out of this util class and change to use DelegateCredentialsProvider
63-
/**
64-
* Adapter between the sdk credentials provider and the crt credentials provider.
65-
*/
66-
public static CredentialsProvider createCrtCredentialsProvider(AwsCredentialsProvider awsCredentialsProvider) {
67-
AwsCredentials sdkCredentials = awsCredentialsProvider.resolveCredentials();
68-
StaticCredentialsProvider.StaticCredentialsProviderBuilder builder =
69-
new StaticCredentialsProvider.StaticCredentialsProviderBuilder();
70-
71-
if (sdkCredentials instanceof AwsSessionCredentials) {
72-
builder.withSessionToken(((AwsSessionCredentials) sdkCredentials).sessionToken().getBytes(StandardCharsets.UTF_8));
73-
}
74-
75-
return builder.withAccessKeyId(sdkCredentials.accessKeyId().getBytes(StandardCharsets.UTF_8))
76-
.withSecretAccessKey(sdkCredentials.secretAccessKey().getBytes(StandardCharsets.UTF_8))
77-
.build();
78-
}
79-
8056
public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetObjectRequest request) {
8157
com.amazonaws.s3.model.GetObjectRequest.Builder getObjectBuilder =
8258
com.amazonaws.s3.model.GetObjectRequest.builder()
@@ -110,7 +86,6 @@ public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetO
11086

11187
}
11288

113-
// TODO: codegen and add tests
11489
public static GetObjectResponse fromCrtGetObjectOutput(GetObjectOutput response, SdkHttpResponse sdkHttpResponse) {
11590
S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse);
11691

@@ -236,7 +211,6 @@ public static com.amazonaws.s3.model.PutObjectRequest toCrtPutObjectRequest(PutO
236211
return putObjectBuilder.build();
237212
}
238213

239-
// TODO: codegen and add tests
240214
public static PutObjectResponse fromCrtPutObjectOutput(PutObjectOutput crtPutObjectOutput) {
241215
// TODO: Provide the HTTP request-level data (e.g. response metadata, HTTP response)
242216
PutObjectResponse.Builder builder = PutObjectResponse.builder()

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717

1818

1919
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
21+
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
2022
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
21-
import software.amazon.awssdk.crt.auth.credentials.DefaultChainCredentialsProvider;
2223
import software.amazon.awssdk.crt.io.ClientBootstrap;
2324
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
2425
import software.amazon.awssdk.transfer.s3.SizeConstant;
@@ -33,6 +34,7 @@ public final class S3NativeClientConfiguration implements SdkAutoCloseable {
3334
private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 5;
3435
private final String signingRegion;
3536
private final ClientBootstrap clientBootstrap;
37+
private final CrtCredentialsProviderAdapter credentialProviderAdapter;
3638
private final CredentialsProvider credentialsProvider;
3739
private final long partSizeInBytes;
3840
private final double targetThroughputInGbps;
@@ -42,11 +44,12 @@ public S3NativeClientConfiguration(Builder builder) {
4244
this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
4345
builder.signingRegion;
4446
this.clientBootstrap = new ClientBootstrap(null, null);
45-
this.credentialsProvider = builder.credentialsProvider == null ?
46-
new DefaultChainCredentialsProvider.DefaultChainCredentialsProviderBuilder()
47-
.withClientBootstrap(clientBootstrap)
48-
.build() :
49-
builder.credentialsProvider;
47+
this.credentialProviderAdapter =
48+
builder.credentialsProvider == null ?
49+
new CrtCredentialsProviderAdapter(DefaultCredentialsProvider.create()) :
50+
new CrtCredentialsProviderAdapter(builder.credentialsProvider);
51+
this.credentialsProvider = credentialProviderAdapter.crtCredentials();
52+
5053
this.partSizeInBytes = builder.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES :
5154
builder.partSizeInBytes;
5255
this.targetThroughputInGbps = builder.targetThroughputInGbps == null ?
@@ -87,12 +90,13 @@ public int maxConcurrency() {
8790
@Override
8891
public void close() {
8992
clientBootstrap.close();
93+
credentialProviderAdapter.close();
9094
credentialsProvider.close();
9195
}
9296

9397
public static final class Builder {
9498
private String signingRegion;
95-
private CredentialsProvider credentialsProvider;
99+
private AwsCredentialsProvider credentialsProvider;
96100
private Long partSizeInBytes;
97101
private Double targetThroughputInGbps;
98102
private Integer maxConcurrency;
@@ -105,7 +109,7 @@ public Builder signingRegion(String signingRegion) {
105109
return this;
106110
}
107111

108-
public Builder credentialsProvider(CredentialsProvider credentialsProvider) {
112+
public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
109113
this.credentialsProvider = credentialsProvider;
110114
return this;
111115
}

‎services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialProviderAdapterTest.java

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3.internal;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.mockito.Mockito.verify;
20+
import static org.mockito.Mockito.when;
21+
22+
import java.nio.charset.StandardCharsets;
23+
import org.junit.Test;
24+
import org.mockito.Mockito;
25+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
26+
import software.amazon.awssdk.auth.credentials.AwsCredentials;
27+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
28+
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
29+
import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider;
30+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
31+
import software.amazon.awssdk.crt.auth.credentials.Credentials;
32+
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
33+
34+
public class CrtCredentialProviderAdapterTest {
35+
36+
@Test
37+
public void crtCredentials_withSession_shouldConvert() {
38+
AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
39+
.create(AwsSessionCredentials.create("foo", "bar", "session"));
40+
41+
CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider)
42+
.crtCredentials();
43+
44+
Credentials credentials = crtCredentialsProvider.getCredentials().join();
45+
46+
assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
47+
assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
48+
assertThat(credentials.getSessionToken()).isEqualTo("session".getBytes(StandardCharsets.UTF_8));
49+
}
50+
51+
@Test
52+
public void crtCredentials_withoutSession_shouldConvert() {
53+
AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
54+
.create(AwsBasicCredentials.create("foo", "bar"));
55+
56+
CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider)
57+
.crtCredentials();
58+
59+
Credentials credentials = crtCredentialsProvider.getCredentials().join();
60+
61+
assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
62+
assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
63+
assertThat(credentials.getSessionToken()).isNull();
64+
}
65+
66+
@Test
67+
public void crtCredentials_provideAwsCredentials_shouldInvokeResolveAndClose() {
68+
HttpCredentialsProvider awsCredentialsProvider = Mockito.mock(HttpCredentialsProvider.class);
69+
AwsCredentials credentials = new AwsCredentials() {
70+
@Override
71+
public String accessKeyId() {
72+
return "foo";
73+
}
74+
75+
@Override
76+
public String secretAccessKey() {
77+
return "bar";
78+
}
79+
};
80+
when(awsCredentialsProvider.resolveCredentials()).thenReturn(credentials);
81+
82+
CrtCredentialsProviderAdapter adapter = new CrtCredentialsProviderAdapter(awsCredentialsProvider);
83+
CredentialsProvider crtCredentialsProvider = adapter.crtCredentials();
84+
85+
Credentials crtCredentials = crtCredentialsProvider.getCredentials().join();
86+
assertThat(crtCredentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8));
87+
assertThat(crtCredentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8));
88+
verify(awsCredentialsProvider).resolveCredentials();
89+
90+
adapter.close();
91+
verify(awsCredentialsProvider).close();
92+
}
93+
}

‎services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,28 +23,21 @@
2323
import com.amazonaws.s3.model.PutObjectOutput;
2424
import com.amazonaws.s3.model.ReplicationStatus;
2525
import java.lang.reflect.Field;
26-
import java.nio.charset.StandardCharsets;
2726
import java.time.Duration;
2827
import java.time.Instant;
2928
import java.util.Arrays;
3029
import java.util.Collection;
3130
import java.util.HashMap;
3231
import java.util.Map;
3332
import java.util.Random;
34-
import java.util.concurrent.ExecutionException;
3533
import org.apache.commons.lang3.RandomStringUtils;
3634
import org.junit.Test;
3735
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
38-
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
39-
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
40-
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
4136
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
4237
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
4338
import software.amazon.awssdk.core.ApiName;
4439
import software.amazon.awssdk.core.SdkField;
4540
import software.amazon.awssdk.core.util.SdkUserAgent;
46-
import software.amazon.awssdk.crt.auth.credentials.Credentials;
47-
import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider;
4841
import software.amazon.awssdk.crt.http.HttpHeader;
4942
import software.amazon.awssdk.http.SdkHttpResponse;
5043
import software.amazon.awssdk.metrics.LoggingMetricPublisher;
@@ -67,19 +60,6 @@ public class S3CrtPojoConversionTest {
6760
private static final String SECRET_ACCESS_KEY = "secretAccessKey";
6861
private static final String SESSION_TOKEN = "sessionToken";
6962

70-
@Test
71-
public void createCrtCredentialsProviderTest() throws ExecutionException, InterruptedException {
72-
AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
73-
.create(AwsSessionCredentials.create(ACCESS_KEY, SECRET_ACCESS_KEY, SESSION_TOKEN));
74-
CredentialsProvider crtCredentialsProvider = S3CrtPojoConversion.createCrtCredentialsProvider(awsCredentialsProvider);
75-
76-
Credentials credentials = crtCredentialsProvider.getCredentials().get();
77-
78-
assertThat(ACCESS_KEY.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getAccessKeyId());
79-
assertThat(SECRET_ACCESS_KEY.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getSecretAccessKey());
80-
assertThat(SESSION_TOKEN.getBytes(StandardCharsets.UTF_8)).isEqualTo(credentials.getSessionToken());
81-
}
82-
8363
@Test
8464
public void fromCrtPutObjectOutputAllFields_shouldConvert() throws IllegalAccessException {
8565

0 commit comments

Comments
 (0)