Skip to content

Commit f1576c3

Browse files
zoewanggzoewangg
committed
Integrate with CRT S3 express support
1 parent 0340939 commit f1576c3

File tree

7 files changed

+230
-41
lines changed

7 files changed

+230
-41
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@
116116
<rxjava.version>2.2.21</rxjava.version>
117117
<commons-codec.verion>1.15</commons-codec.verion>
118118
<jmh.version>1.29</jmh.version>
119-
<awscrt.version>0.28.0</awscrt.version>
119+
<awscrt.version>0.28.10</awscrt.version>
120120

121121
<!--Test dependencies -->
122122
<junit5.version>5.10.0</junit5.version>

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CrtS3ExpressNoOpAuthScheme.java

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.internal.crt;
17+
18+
import java.util.concurrent.CompletableFuture;
19+
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
21+
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest;
22+
import software.amazon.awssdk.http.auth.spi.signer.HttpSigner;
23+
import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
24+
import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
25+
import software.amazon.awssdk.identity.spi.IdentityProvider;
26+
import software.amazon.awssdk.identity.spi.IdentityProviders;
27+
import software.amazon.awssdk.identity.spi.ResolveIdentityRequest;
28+
import software.amazon.awssdk.services.s3.s3express.S3ExpressAuthScheme;
29+
import software.amazon.awssdk.services.s3.s3express.S3ExpressSessionCredentials;
30+
31+
/**
32+
* An implementation of {@link S3ExpressAuthScheme} that returns a noop {@link IdentityProvider}.
33+
*/
34+
@SdkInternalApi
35+
public final class CrtS3ExpressNoOpAuthScheme implements S3ExpressAuthScheme {
36+
@Override
37+
public String schemeId() {
38+
return S3ExpressAuthScheme.SCHEME_ID;
39+
}
40+
41+
@Override
42+
public IdentityProvider<S3ExpressSessionCredentials> identityProvider(IdentityProviders providers) {
43+
return NoOpIdentityProvider.INSTANCE;
44+
}
45+
46+
@Override
47+
public HttpSigner<S3ExpressSessionCredentials> signer() {
48+
return NoOpSigner.INSTANCE;
49+
}
50+
51+
private static final class NoOpIdentityProvider implements IdentityProvider<S3ExpressSessionCredentials> {
52+
private static final NoOpIdentityProvider INSTANCE = new NoOpIdentityProvider();
53+
54+
@Override
55+
public Class<S3ExpressSessionCredentials> identityType() {
56+
return S3ExpressSessionCredentials.class;
57+
}
58+
59+
@Override
60+
public CompletableFuture<? extends S3ExpressSessionCredentials> resolveIdentity(ResolveIdentityRequest request) {
61+
return CompletableFuture.completedFuture(null);
62+
}
63+
}
64+
65+
private static final class NoOpSigner implements HttpSigner<S3ExpressSessionCredentials> {
66+
private static final NoOpSigner INSTANCE = new NoOpSigner();
67+
68+
@Override
69+
public SignedRequest sign(SignRequest<? extends S3ExpressSessionCredentials> request) {
70+
throw new UnsupportedOperationException();
71+
}
72+
73+
@Override
74+
public CompletableFuture<AsyncSignedRequest> signAsync(AsyncSignRequest<? extends S3ExpressSessionCredentials> request) {
75+
throw new UnsupportedOperationException();
76+
}
77+
}
78+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,14 @@
1515

1616
package software.amazon.awssdk.services.s3.internal.crt;
1717

18+
import static software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME;
1819
import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.AUTH_SCHEMES;
1920
import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SDK_HTTP_EXECUTION_ATTRIBUTES;
2021
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.HTTP_CHECKSUM;
2122
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME;
23+
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_NAME;
2224
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_REGION;
25+
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.USE_S3_EXPRESS_AUTH;
2326
import static software.amazon.awssdk.services.s3.internal.crt.S3NativeClientConfiguration.DEFAULT_PART_SIZE_IN_BYTES;
2427

2528
import java.net.URI;
@@ -58,6 +61,7 @@
5861
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
5962
import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration;
6063
import software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper;
64+
import software.amazon.awssdk.services.s3.internal.s3express.S3ExpressUtils;
6165
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
6266
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
6367
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -129,6 +133,7 @@ private static S3AsyncClient initializeS3AsyncClient(DefaultS3CrtClientBuilder b
129133
.accelerate(builder.accelerate)
130134
.forcePathStyle(builder.forcePathStyle)
131135
.crossRegionAccessEnabled(builder.crossRegionAccessEnabled)
136+
.putAuthScheme(new CrtS3ExpressNoOpAuthScheme())
132137
.httpClientBuilder(initializeS3CrtAsyncHttpClient(builder))
133138
.build();
134139
}
@@ -309,6 +314,8 @@ public void afterMarshalling(Context.AfterMarshalling context,
309314
.put(SIGNING_REGION, executionAttributes.getAttribute(AwsSignerExecutionAttribute.SIGNING_REGION))
310315
.put(S3InternalSdkHttpExecutionAttribute.OBJECT_FILE_PATH,
311316
executionAttributes.getAttribute(OBJECT_FILE_PATH))
317+
.put(USE_S3_EXPRESS_AUTH, S3ExpressUtils.useS3ExpressAuthScheme(executionAttributes))
318+
.put(SIGNING_NAME, executionAttributes.getAttribute(SERVICE_SIGNING_NAME))
312319
.build();
313320

314321
// For putObject and getObject, we rely on CRT to perform checksum validation

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java

Lines changed: 57 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.HTTP_CHECKSUM;
2323
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.OBJECT_FILE_PATH;
2424
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME;
25+
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_NAME;
2526
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.SIGNING_REGION;
27+
import static software.amazon.awssdk.services.s3.internal.crt.S3InternalSdkHttpExecutionAttribute.USE_S3_EXPRESS_AUTH;
2628
import static software.amazon.awssdk.utils.FunctionalUtils.invokeSafely;
2729

2830
import java.net.URI;
@@ -46,6 +48,7 @@
4648
import software.amazon.awssdk.crt.s3.S3MetaRequest;
4749
import software.amazon.awssdk.crt.s3.S3MetaRequestOptions;
4850
import software.amazon.awssdk.http.Header;
51+
import software.amazon.awssdk.http.SdkHttpExecutionAttributes;
4952
import software.amazon.awssdk.http.SdkHttpRequest;
5053
import software.amazon.awssdk.http.async.AsyncExecuteRequest;
5154
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
@@ -70,9 +73,28 @@ public final class S3CrtAsyncHttpClient implements SdkAsyncHttpClient {
7073

7174
private S3CrtAsyncHttpClient(Builder builder) {
7275
s3NativeClientConfiguration = builder.clientConfiguration;
76+
this.s3ClientOptions = createS3ClientOption();
77+
78+
this.crtS3Client = new S3Client(s3ClientOptions);
79+
}
80+
81+
@SdkTestInternalApi
82+
S3CrtAsyncHttpClient(S3Client crtS3Client,
83+
Builder builder) {
84+
s3NativeClientConfiguration = builder.clientConfiguration;
85+
s3ClientOptions = createS3ClientOption();
86+
this.crtS3Client = crtS3Client;
87+
}
88+
89+
@SdkTestInternalApi
90+
public S3ClientOptions s3ClientOptions() {
91+
return s3ClientOptions;
92+
}
93+
94+
private S3ClientOptions createS3ClientOption() {
7395
Long initialWindowSize = s3NativeClientConfiguration.readBufferSizeInBytes();
7496

75-
this.s3ClientOptions =
97+
S3ClientOptions options =
7698
new S3ClientOptions().withRegion(s3NativeClientConfiguration.signingRegion())
7799
.withEndpoint(s3NativeClientConfiguration.endpointOverride() == null ? null :
78100
s3NativeClientConfiguration.endpointOverride().toString())
@@ -82,80 +104,64 @@ private S3CrtAsyncHttpClient(Builder builder) {
82104
.withPartSize(s3NativeClientConfiguration.partSizeBytes())
83105
.withMultipartUploadThreshold(s3NativeClientConfiguration.thresholdInBytes())
84106
.withComputeContentMd5(false)
107+
.withEnableS3Express(true)
85108
.withMaxConnections(s3NativeClientConfiguration.maxConcurrency())
86109
.withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps())
87110
.withInitialReadWindowSize(initialWindowSize)
88111
.withReadBackpressureEnabled(true);
89112

90113
if (s3NativeClientConfiguration.standardRetryOptions() != null) {
91-
this.s3ClientOptions.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions());
114+
options.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions());
92115
}
93116
if (Boolean.FALSE.equals(s3NativeClientConfiguration.isUseEnvironmentVariableValues())) {
94-
s3ClientOptions.withProxyEnvironmentVariableSetting(disabledHttpProxyEnvironmentVariableSetting());
117+
options.withProxyEnvironmentVariableSetting(disabledHttpProxyEnvironmentVariableSetting());
95118
}
96-
Optional.ofNullable(s3NativeClientConfiguration.proxyOptions()).ifPresent(s3ClientOptions::withProxyOptions);
119+
Optional.ofNullable(s3NativeClientConfiguration.proxyOptions()).ifPresent(options::withProxyOptions);
97120
Optional.ofNullable(s3NativeClientConfiguration.connectionTimeout())
98121
.map(Duration::toMillis)
99122
.map(NumericUtils::saturatedCast)
100-
.ifPresent(s3ClientOptions::withConnectTimeoutMs);
123+
.ifPresent(options::withConnectTimeoutMs);
101124
Optional.ofNullable(s3NativeClientConfiguration.httpMonitoringOptions())
102-
.ifPresent(s3ClientOptions::withHttpMonitoringOptions);
103-
104-
this.crtS3Client = new S3Client(s3ClientOptions);
105-
}
106-
107-
@SdkTestInternalApi
108-
S3CrtAsyncHttpClient(S3Client crtS3Client,
109-
S3NativeClientConfiguration nativeClientConfiguration) {
110-
this.crtS3Client = crtS3Client;
111-
this.s3NativeClientConfiguration = nativeClientConfiguration;
112-
this.s3ClientOptions = null;
113-
}
114-
115-
@SdkTestInternalApi
116-
public S3ClientOptions s3ClientOptions() {
117-
return s3ClientOptions;
125+
.ifPresent(options::withHttpMonitoringOptions);
126+
return options;
118127
}
119128

120129
@Override
121130
public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
122131
CompletableFuture<Void> executeFuture = new CompletableFuture<>();
123132
URI uri = asyncRequest.request().getUri();
124133
HttpRequest httpRequest = toCrtRequest(asyncRequest);
134+
SdkHttpExecutionAttributes httpExecutionAttributes = asyncRequest.httpExecutionAttributes();
125135
S3CrtResponseHandlerAdapter responseHandler =
126136
new S3CrtResponseHandlerAdapter(executeFuture,
127137
asyncRequest.responseHandler(),
128-
asyncRequest.httpExecutionAttributes().getAttribute(CRT_PROGRESS_LISTENER));
138+
httpExecutionAttributes.getAttribute(CRT_PROGRESS_LISTENER));
129139

130140
S3MetaRequestOptions.MetaRequestType requestType = requestType(asyncRequest);
131141

132-
HttpChecksum httpChecksum = asyncRequest.httpExecutionAttributes().getAttribute(HTTP_CHECKSUM);
133-
ResumeToken resumeToken = asyncRequest.httpExecutionAttributes().getAttribute(CRT_PAUSE_RESUME_TOKEN);
134-
Region signingRegion = asyncRequest.httpExecutionAttributes().getAttribute(SIGNING_REGION);
135-
Path requestFilePath = asyncRequest.httpExecutionAttributes().getAttribute(OBJECT_FILE_PATH);
142+
HttpChecksum httpChecksum = httpExecutionAttributes.getAttribute(HTTP_CHECKSUM);
143+
ResumeToken resumeToken = httpExecutionAttributes.getAttribute(CRT_PAUSE_RESUME_TOKEN);
144+
Region signingRegion = httpExecutionAttributes.getAttribute(SIGNING_REGION);
145+
Path requestFilePath = httpExecutionAttributes.getAttribute(OBJECT_FILE_PATH);
136146
ChecksumConfig checksumConfig =
137147
checksumConfig(httpChecksum, requestType, s3NativeClientConfiguration.checksumValidationEnabled());
138148
URI endpoint = getEndpoint(uri);
139149

150+
AwsSigningConfig defaultS3SigningConfig = awsSigningConfig(signingRegion, httpExecutionAttributes);
151+
140152
S3MetaRequestOptions requestOptions = new S3MetaRequestOptions()
141153
.withHttpRequest(httpRequest)
142154
.withMetaRequestType(requestType)
143155
.withChecksumConfig(checksumConfig)
144156
.withEndpoint(endpoint)
145157
.withResponseHandler(responseHandler)
146158
.withResumeToken(resumeToken)
147-
.withRequestFilePath(requestFilePath);
148-
149-
// Create a new SigningConfig object only if the signing region has changed from the previously configured region.
150-
if (signingRegion != null && !s3ClientOptions.getRegion().equals(signingRegion.id())) {
151-
requestOptions.withSigningConfig(
152-
AwsSigningConfig.getDefaultS3SigningConfig(signingRegion.id(),
153-
s3ClientOptions.getCredentialsProvider()));
154-
}
159+
.withRequestFilePath(requestFilePath)
160+
.withSigningConfig(defaultS3SigningConfig);
155161

156162
S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions);
157163
S3MetaRequestPauseObservable observable =
158-
asyncRequest.httpExecutionAttributes().getAttribute(METAREQUEST_PAUSE_OBSERVABLE);
164+
httpExecutionAttributes.getAttribute(METAREQUEST_PAUSE_OBSERVABLE);
159165

160166
responseHandler.metaRequest(s3MetaRequest);
161167

@@ -167,6 +173,22 @@ public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
167173
return executeFuture;
168174
}
169175

176+
private AwsSigningConfig awsSigningConfig(Region signingRegion, SdkHttpExecutionAttributes httpExecutionAttributes) {
177+
AwsSigningConfig defaultS3SigningConfig =
178+
AwsSigningConfig.getDefaultS3SigningConfig(s3ClientOptions.getRegion(), s3ClientOptions.getCredentialsProvider());
179+
180+
// Override the region only if the signing region has changed from the previously configured region.
181+
if (signingRegion != null && !s3ClientOptions.getRegion().equals(signingRegion.id())) {
182+
defaultS3SigningConfig.setRegion(signingRegion.id());
183+
}
184+
185+
defaultS3SigningConfig.setService(httpExecutionAttributes.getAttribute(SIGNING_NAME));
186+
187+
if (Boolean.TRUE.equals(httpExecutionAttributes.getAttribute(USE_S3_EXPRESS_AUTH))) {
188+
defaultS3SigningConfig.setAlgorithm(AwsSigningConfig.AwsSigningAlgorithm.SIGV4_S3EXPRESS);
189+
}
190+
return defaultS3SigningConfig;
191+
}
170192

171193
private static URI getEndpoint(URI uri) {
172194
return invokeSafely(() -> new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), null, null, null));

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3InternalSdkHttpExecutionAttribute.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,14 @@ public final class S3InternalSdkHttpExecutionAttribute<T> extends SdkHttpExecuti
4040
public static final S3InternalSdkHttpExecutionAttribute<Region> SIGNING_REGION =
4141
new S3InternalSdkHttpExecutionAttribute<>(Region.class);
4242

43+
public static final S3InternalSdkHttpExecutionAttribute<String> SIGNING_NAME =
44+
new S3InternalSdkHttpExecutionAttribute<>(String.class);
45+
4346
public static final S3InternalSdkHttpExecutionAttribute<Path> OBJECT_FILE_PATH =
4447
new S3InternalSdkHttpExecutionAttribute<>(Path.class);
4548

49+
public static final S3InternalSdkHttpExecutionAttribute<Boolean> USE_S3_EXPRESS_AUTH =
50+
new S3InternalSdkHttpExecutionAttribute<>(Boolean.class);
4651

4752
private S3InternalSdkHttpExecutionAttribute(Class<T> valueClass) {
4853
super(valueClass);

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3express/S3ExpressUtils.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,16 @@
1515

1616
package software.amazon.awssdk.services.s3.internal.s3express;
1717

18+
import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SELECTED_AUTH_SCHEME;
19+
1820
import software.amazon.awssdk.annotations.SdkInternalApi;
21+
import software.amazon.awssdk.core.SelectedAuthScheme;
1922
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
2023
import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute;
2124
import software.amazon.awssdk.endpoints.Endpoint;
25+
import software.amazon.awssdk.http.auth.spi.scheme.AuthSchemeOption;
2226
import software.amazon.awssdk.services.s3.endpoints.internal.KnownS3ExpressEndpointProperty;
27+
import software.amazon.awssdk.services.s3.s3express.S3ExpressAuthScheme;
2328

2429
@SdkInternalApi
2530
public final class S3ExpressUtils {
@@ -40,4 +45,16 @@ public static boolean useS3Express(ExecutionAttributes executionAttributes) {
4045
}
4146
return false;
4247
}
48+
49+
/**
50+
* Whether aws.auth#sigv4-s3express is used or not
51+
*/
52+
public static boolean useS3ExpressAuthScheme(ExecutionAttributes executionAttributes) {
53+
SelectedAuthScheme<?> selectedAuthScheme = executionAttributes.getAttribute(SELECTED_AUTH_SCHEME);
54+
if (selectedAuthScheme != null) {
55+
AuthSchemeOption authSchemeOption = selectedAuthScheme.authSchemeOption();
56+
return S3ExpressAuthScheme.SCHEME_ID.equals(authSchemeOption.schemeId());
57+
}
58+
return false;
59+
}
4360
}

0 commit comments

Comments
 (0)