Skip to content

Commit c305d38

Browse files
joviegasjoviegas
authored
Expose numRetries to configure maxRetries on error in the S3CrtClient Interface (#3885)
* Expose StandardRetryOptions in the S3CrtClient Interface * Added wrapper class for CRT retry config * Added Null checks and test cases * Handled PR comment on javadoc * Handled NIT comments
1 parent 9b4dc8f commit c305d38

File tree

8 files changed

+230
-4
lines changed

8 files changed

+230
-4
lines changed

‎.changes/next-release/feature-AWSSDKforJavav2-ddc21df.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "AWS SDK for Java v2",
4+
"contributor": "",
5+
"description": "Expose StandardRetryOptions of S3ClientOptions in the S3CrtClient Interface"
6+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
2323
import software.amazon.awssdk.regions.Region;
2424
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
25+
import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration;
2526
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
2627
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
2728
import software.amazon.awssdk.utils.Validate;
@@ -161,6 +162,14 @@ public interface S3CrtAsyncClientBuilder extends SdkBuilder<S3CrtAsyncClientBuil
161162
*/
162163
S3CrtAsyncClientBuilder httpConfiguration(S3CrtHttpConfiguration configuration);
163164

165+
/**
166+
* Sets the Retry configuration to use for this client.
167+
*
168+
* @param retryConfiguration The retry configurations to be used.
169+
* @return The builder of the method chaining.
170+
*/
171+
S3CrtAsyncClientBuilder retryConfiguration(S3CrtRetryConfiguration retryConfiguration);
172+
164173
/**
165174
* A convenience method that creates an instance of the {@link S3CrtHttpConfiguration} builder, avoiding the
166175
* need to create one manually via {@link S3CrtHttpConfiguration#builder()}.
@@ -188,6 +197,25 @@ default S3CrtAsyncClientBuilder httpConfiguration(Consumer<S3CrtHttpConfiguratio
188197
*/
189198
S3CrtAsyncClientBuilder forcePathStyle(Boolean forcePathStyle);
190199

200+
/**
201+
* A convenience method that creates an instance of the {@link S3CrtRetryConfiguration} builder, avoiding the
202+
* need to create one manually via {@link S3CrtRetryConfiguration#builder()}.
203+
*
204+
* @param retryConfigurationBuilder The retry config builder to use
205+
* @return The builder of the method chaining.
206+
* @see #retryConfiguration(S3CrtRetryConfiguration)
207+
*/
208+
default S3CrtAsyncClientBuilder retryConfiguration(Consumer<S3CrtRetryConfiguration.Builder> retryConfigurationBuilder) {
209+
Validate.paramNotNull(retryConfigurationBuilder, "retryConfigurationBuilder");
210+
return retryConfiguration(S3CrtRetryConfiguration.builder()
211+
.applyMutation(retryConfigurationBuilder)
212+
.build());
213+
}
214+
215+
216+
217+
218+
191219
@Override
192220
S3AsyncClient build();
193221
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtRetryConfiguration.java

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.crt;
17+
18+
import java.util.Objects;
19+
import software.amazon.awssdk.annotations.Immutable;
20+
import software.amazon.awssdk.annotations.SdkPublicApi;
21+
import software.amazon.awssdk.annotations.ThreadSafe;
22+
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
23+
import software.amazon.awssdk.utils.Validate;
24+
import software.amazon.awssdk.utils.builder.CopyableBuilder;
25+
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
26+
27+
/**
28+
* Retry option configuration for AWS CRT-based S3 client.
29+
*
30+
* @see S3CrtAsyncClientBuilder#retryConfiguration
31+
*/
32+
@SdkPublicApi
33+
@Immutable
34+
@ThreadSafe
35+
public final class S3CrtRetryConfiguration implements ToCopyableBuilder<S3CrtRetryConfiguration.Builder,
36+
S3CrtRetryConfiguration> {
37+
private final Integer numRetries;
38+
39+
private S3CrtRetryConfiguration(DefaultBuilder builder) {
40+
Validate.notNull(builder.numRetries, "numRetries");
41+
this.numRetries = builder.numRetries;
42+
}
43+
44+
/**
45+
* Creates a default builder for {@link S3CrtRetryConfiguration}.
46+
*/
47+
public static Builder builder() {
48+
return new S3CrtRetryConfiguration.DefaultBuilder();
49+
}
50+
51+
/**
52+
* Retrieve the {@link S3CrtRetryConfiguration.Builder#numRetries(Integer)} configured on the builder.
53+
*/
54+
public Integer numRetries() {
55+
return numRetries;
56+
}
57+
58+
@Override
59+
public boolean equals(Object o) {
60+
if (this == o) {
61+
return true;
62+
}
63+
if (o == null || getClass() != o.getClass()) {
64+
return false;
65+
}
66+
S3CrtRetryConfiguration that = (S3CrtRetryConfiguration) o;
67+
return Objects.equals(numRetries, that.numRetries);
68+
}
69+
70+
@Override
71+
public int hashCode() {
72+
return numRetries != null ? numRetries.hashCode() : 0;
73+
}
74+
75+
@Override
76+
public Builder toBuilder() {
77+
return new S3CrtRetryConfiguration.DefaultBuilder(this);
78+
}
79+
80+
public interface Builder extends CopyableBuilder<S3CrtRetryConfiguration.Builder, S3CrtRetryConfiguration> {
81+
82+
/**
83+
* Sets the maximum number of retries for a single HTTP request.
84+
* <p> For example, if an upload operation is split into 5 HTTP service requests ( One for initiate, Three for
85+
* uploadPart and one for completeUpload), then numRetries specifies the maximum number of retries for each failed
86+
* request, not for the entire uploadObject operation.
87+
*
88+
* @param numRetries The maximum number of retries for a single HTTP request.
89+
* @return The builder of the method chaining.
90+
*/
91+
Builder numRetries(Integer numRetries);
92+
93+
}
94+
95+
private static final class DefaultBuilder implements Builder {
96+
private Integer numRetries;
97+
98+
private DefaultBuilder() {
99+
}
100+
101+
private DefaultBuilder(S3CrtRetryConfiguration crtRetryConfiguration) {
102+
this.numRetries = crtRetryConfiguration.numRetries;
103+
}
104+
105+
@Override
106+
public Builder numRetries(Integer numRetries) {
107+
this.numRetries = numRetries;
108+
return this;
109+
}
110+
111+
@Override
112+
public S3CrtRetryConfiguration build() {
113+
return new S3CrtRetryConfiguration(this);
114+
}
115+
}
116+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,16 @@
4141
import software.amazon.awssdk.core.internal.util.ClassLoaderHelper;
4242
import software.amazon.awssdk.core.retry.RetryPolicy;
4343
import software.amazon.awssdk.core.signer.NoOpSigner;
44+
import software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions;
45+
import software.amazon.awssdk.crt.io.StandardRetryOptions;
4446
import software.amazon.awssdk.http.SdkHttpExecutionAttributes;
4547
import software.amazon.awssdk.regions.Region;
4648
import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient;
4749
import software.amazon.awssdk.services.s3.S3AsyncClient;
4850
import software.amazon.awssdk.services.s3.S3Configuration;
4951
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
5052
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
53+
import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration;
5154
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
5255
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
5356
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -109,7 +112,7 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau
109112
Validate.isPositiveOrNull(builder.targetThroughputInGbps, "targetThroughputInGbps");
110113
Validate.isPositiveOrNull(builder.minimalPartSizeInBytes, "minimalPartSizeInBytes");
111114

112-
S3NativeClientConfiguration s3NativeClientConfiguration =
115+
S3NativeClientConfiguration.Builder nativeClientBuilder =
113116
S3NativeClientConfiguration.builder()
114117
.checksumValidationEnabled(builder.checksumValidationEnabled)
115118
.targetThroughputInGbps(builder.targetThroughputInGbps)
@@ -119,11 +122,16 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau
119122
.endpointOverride(builder.endpointOverride)
120123
.credentialsProvider(builder.credentialsProvider)
121124
.readBufferSizeInBytes(builder.readBufferSizeInBytes)
122-
.httpConfiguration(builder.httpConfiguration)
123-
.build();
125+
.httpConfiguration(builder.httpConfiguration);
124126

127+
if (builder.retryConfiguration != null) {
128+
nativeClientBuilder.standardRetryOptions(
129+
new StandardRetryOptions()
130+
.withBackoffRetryOptions(new ExponentialBackoffRetryOptions()
131+
.withMaxRetries(builder.retryConfiguration.numRetries())));
132+
}
125133
return S3CrtAsyncHttpClient.builder()
126-
.s3ClientConfiguration(s3NativeClientConfiguration);
134+
.s3ClientConfiguration(nativeClientBuilder.build());
127135
}
128136

129137
public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder {
@@ -140,6 +148,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
140148
private Boolean forcePathStyle;
141149

142150
private List<ExecutionInterceptor> executionInterceptors;
151+
private S3CrtRetryConfiguration retryConfiguration;
143152

144153
public AwsCredentialsProvider credentialsProvider() {
145154
return credentialsProvider;
@@ -244,6 +253,12 @@ S3CrtAsyncClientBuilder addExecutionInterceptor(ExecutionInterceptor executionIn
244253
return this;
245254
}
246255

256+
@Override
257+
public S3CrtAsyncClientBuilder retryConfiguration(S3CrtRetryConfiguration retryConfiguration) {
258+
this.retryConfiguration = retryConfiguration;
259+
return this;
260+
}
261+
247262
@Override
248263
public S3CrtAsyncClient build() {
249264
return new DefaultS3CrtAsyncClient(this);

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ private S3CrtAsyncHttpClient(Builder builder) {
7878
.withInitialReadWindowSize(initialWindowSize)
7979
.withReadBackpressureEnabled(true);
8080

81+
if (s3NativeClientConfiguration.standardRetryOptions() != null) {
82+
this.s3ClientOptions.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions());
83+
}
8184
Optional.ofNullable(s3NativeClientConfiguration.proxyOptions()).ifPresent(s3ClientOptions::withProxyOptions);
8285
Optional.ofNullable(s3NativeClientConfiguration.connectionTimeout())
8386
.map(Duration::toMillis)

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import software.amazon.awssdk.crt.http.HttpMonitoringOptions;
2828
import software.amazon.awssdk.crt.http.HttpProxyOptions;
2929
import software.amazon.awssdk.crt.io.ClientBootstrap;
30+
import software.amazon.awssdk.crt.io.StandardRetryOptions;
3031
import software.amazon.awssdk.crt.io.TlsCipherPreference;
3132
import software.amazon.awssdk.crt.io.TlsContext;
3233
import software.amazon.awssdk.crt.io.TlsContextOptions;
@@ -43,6 +44,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
4344
private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 10;
4445

4546
private final String signingRegion;
47+
private final StandardRetryOptions standardRetryOptions;
4648
private final ClientBootstrap clientBootstrap;
4749
private final CrtCredentialsProviderAdapter credentialProviderAdapter;
4850
private final CredentialsProvider credentialsProvider;
@@ -97,6 +99,7 @@ public S3NativeClientConfiguration(Builder builder) {
9799
this.connectionTimeout = null;
98100
this.httpMonitoringOptions = null;
99101
}
102+
this.standardRetryOptions = builder.standardRetryOptions;
100103
}
101104

102105
public HttpMonitoringOptions httpMonitoringOptions() {
@@ -140,6 +143,10 @@ public int maxConcurrency() {
140143
return maxConcurrency;
141144
}
142145

146+
public StandardRetryOptions standardRetryOptions() {
147+
return standardRetryOptions;
148+
}
149+
143150
public URI endpointOverride() {
144151
return endpointOverride;
145152
}
@@ -169,6 +176,7 @@ public static final class Builder {
169176
private URI endpointOverride;
170177
private Boolean checksumValidationEnabled;
171178
private S3CrtHttpConfiguration httpConfiguration;
179+
private StandardRetryOptions standardRetryOptions;
172180

173181
private Builder() {
174182
}
@@ -224,5 +232,10 @@ public Builder httpConfiguration(S3CrtHttpConfiguration httpConfiguration) {
224232
this.httpConfiguration = httpConfiguration;
225233
return this;
226234
}
235+
236+
public Builder standardRetryOptions(StandardRetryOptions standardRetryOptions) {
237+
this.standardRetryOptions = standardRetryOptions;
238+
return this;
239+
}
227240
}
228241
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/crt/S3CrtRetryConfigurationTest.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.crt;
17+
18+
import nl.jqno.equalsverifier.EqualsVerifier;
19+
import org.junit.jupiter.api.Test;
20+
import static org.assertj.core.api.Assertions.assertThatNullPointerException;
21+
22+
class S3CrtRetryConfigurationTest {
23+
24+
@Test
25+
void equalsHashcode() {
26+
EqualsVerifier.forClass(S3CrtRetryConfiguration.class)
27+
.withRedefinedSuperclass()
28+
.verify();
29+
}
30+
31+
@Test
32+
void retryConfigurationWithNoMaxRetriesDefined(){
33+
assertThatNullPointerException().isThrownBy(() ->S3CrtRetryConfiguration.builder().build())
34+
.withMessage("numRetries");
35+
}
36+
37+
38+
39+
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
import org.mockito.Mockito;
3838
import software.amazon.awssdk.core.interceptor.trait.HttpChecksum;
3939
import software.amazon.awssdk.crt.http.HttpRequest;
40+
import software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions;
41+
import software.amazon.awssdk.crt.io.StandardRetryOptions;
4042
import software.amazon.awssdk.crt.s3.ChecksumAlgorithm;
4143
import software.amazon.awssdk.crt.s3.S3Client;
4244
import software.amazon.awssdk.crt.s3.S3ClientOptions;
@@ -314,6 +316,9 @@ void build_shouldPassThroughParameters() {
314316
S3NativeClientConfiguration.builder()
315317
.maxConcurrency(100)
316318
.signingRegion("us-west-2")
319+
.standardRetryOptions(
320+
new StandardRetryOptions()
321+
.withBackoffRetryOptions(new ExponentialBackoffRetryOptions().withMaxRetries(7)))
317322
.httpConfiguration(S3CrtHttpConfiguration.builder()
318323
.connectionTimeout(Duration.ofSeconds(1))
319324
.connectionHealthConfiguration(c -> c.minimumThroughputInBps(1024L)
@@ -325,6 +330,7 @@ void build_shouldPassThroughParameters() {
325330
(S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build();
326331
S3ClientOptions clientOptions = client.s3ClientOptions();
327332
assertThat(clientOptions.getConnectTimeoutMs()).isEqualTo(1000);
333+
assertThat(clientOptions.getStandardRetryOptions().getBackoffRetryOptions().getMaxRetries()).isEqualTo(7);
328334
assertThat(clientOptions.getMaxConnections()).isEqualTo(100);
329335
assertThat(clientOptions.getMonitoringOptions()).satisfies(options -> {
330336
assertThat(options.getMinThroughputBytesPerSecond()).isEqualTo(1024);

0 commit comments

Comments
 (0)