Skip to content

Add new sync and async retryable stages #4062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions core/aws-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@
<artifactId>utils</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>retries-api</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>retries</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.eventstream</groupId>
<artifactId>eventstream</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import software.amazon.awssdk.awscore.internal.defaultsmode.DefaultsModeConfiguration;
import software.amazon.awssdk.awscore.internal.defaultsmode.DefaultsModeResolver;
import software.amazon.awssdk.awscore.retry.AwsRetryPolicy;
import software.amazon.awssdk.awscore.retry.AwsRetryStrategy;
import software.amazon.awssdk.core.client.builder.SdkDefaultClientBuilder;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
Expand All @@ -53,6 +54,7 @@
import software.amazon.awssdk.regions.ServiceMetadata;
import software.amazon.awssdk.regions.ServiceMetadataAdvancedOption;
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
import software.amazon.awssdk.retries.api.RetryStrategy;
import software.amazon.awssdk.utils.AttributeMap;
import software.amazon.awssdk.utils.CollectionUtils;
import software.amazon.awssdk.utils.Logger;
Expand Down Expand Up @@ -196,6 +198,7 @@ protected final SdkClientConfiguration finalizeChildConfiguration(SdkClientConfi
.option(SdkClientOption.EXECUTION_INTERCEPTORS, addAwsInterceptors(configuration))
.option(AwsClientOption.SIGNING_REGION, resolveSigningRegion(configuration))
.option(SdkClientOption.RETRY_POLICY, resolveAwsRetryPolicy(configuration))
.option(SdkClientOption.RETRY_STRATEGY, resolveAwsRetryStrategy(configuration))
.build();
}

Expand Down Expand Up @@ -375,6 +378,35 @@ private RetryPolicy resolveAwsRetryPolicy(SdkClientConfiguration config) {
.defaultRetryMode(config.option(SdkClientOption.DEFAULT_RETRY_MODE))
.resolve();
return AwsRetryPolicy.forRetryMode(retryMode);
// TODO: fixme This will be changed like this to pick the configured retry strategy
// if no retry policy is configured.
/*
RetryPolicy policy = config.option(SdkClientOption.RETRY_POLICY);

if (policy != null) {
if (policy.additionalRetryConditionsAllowed()) {
return AwsRetryPolicy.addRetryConditions(policy);
} else {
return policy;
}
}

// If we don't have a configured retry policy we will use the configured retry strategy instead.
return null;
*/
}

private RetryStrategy<?, ?> resolveAwsRetryStrategy(SdkClientConfiguration config) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we avoid the Generic wildcard types by referring to the parent classes ?

Copy link
Contributor Author

@sugmanue sugmanue Jun 6, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we cannot. There's no specific or general enough type that we can use here.

RetryStrategy<?, ?> strategy = config.option(SdkClientOption.RETRY_STRATEGY);
if (strategy != null) {
return AwsRetryStrategy.addRetryConditions(strategy);
}
RetryMode retryMode = RetryMode.resolver()
.profileFile(config.option(SdkClientOption.PROFILE_FILE_SUPPLIER))
.profileName(config.option(SdkClientOption.PROFILE_NAME))
.defaultRetryMode(config.option(SdkClientOption.DEFAULT_RETRY_MODE))
.resolve();
return AwsRetryStrategy.forRetryMode(retryMode);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.awscore.retry;

import software.amazon.awssdk.annotations.SdkPublicApi;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.awscore.internal.AwsErrorCode;
import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy;
import software.amazon.awssdk.core.retry.RetryMode;
import software.amazon.awssdk.retries.AdaptiveRetryStrategy;
import software.amazon.awssdk.retries.DefaultRetryStrategy;
import software.amazon.awssdk.retries.LegacyRetryStrategy;
import software.amazon.awssdk.retries.StandardRetryStrategy;
import software.amazon.awssdk.retries.api.RetryStrategy;

/**
* Retry strategies used by clients when communicating with AWS services.
*/
@SdkPublicApi
public final class AwsRetryStrategy {

private AwsRetryStrategy() {
}

/**
* Retrieve the {@link SdkDefaultRetryStrategy#defaultRetryStrategy()} with AWS-specific conditions added.
*
* @return The default retry strategy.
*/
public static RetryStrategy<?, ?> defaultRetryStrategy() {
return forRetryMode(RetryMode.defaultRetryMode());
}

/**
* Retrieve the appropriate retry strategy for the retry mode with AWS-specific conditions added.
*
* @param mode The retry mode for which we want to create a retry strategy.
* @return A retry strategy for the given retry mode.
*/
public static RetryStrategy<?, ?> forRetryMode(RetryMode mode) {
switch (mode) {
case STANDARD:
return standardRetryStrategy();
case ADAPTIVE:
return adaptiveRetryStrategy();
case LEGACY:
return legacyRetryStrategy();
default:
throw new IllegalArgumentException("unknown retry mode: " + mode);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Junit for AwsRetryStrategy and ADAPTIVE case ?

Copy link
Contributor Author

@sugmanue sugmanue Jun 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will add more tests for these after this is merged in a follow up PR; the idea is to make smaller changes to make it easier to review with self-contained minimal changes that build on top of each other.

}
}

/**
* Update the provided {@link RetryStrategy} to add AWS-specific conditions.
*
* @param strategy The strategy to update
* @return The updated strategy
*/
public static RetryStrategy<?, ?> addRetryConditions(RetryStrategy<?, ?> strategy) {
return strategy.toBuilder()
.retryOnException(AwsRetryStrategy::retryOnAwsRetryableErrors)
.build();
}

/**
* Returns a retry strategy that does not retry.
*
* @return A retry strategy that does not retry.
*/
public static RetryStrategy<?, ?> none() {
return DefaultRetryStrategy.none();
}


/**
* Returns a {@link StandardRetryStrategy} with AWS-specific conditions added.
*
* @return A {@link StandardRetryStrategy} with AWS-specific conditions added.
*/
public static StandardRetryStrategy standardRetryStrategy() {
StandardRetryStrategy.Builder builder = SdkDefaultRetryStrategy.standardRetryStrategyBuilder();
return configure(builder).build();
}

/**
* Returns a {@link LegacyRetryStrategy} with AWS-specific conditions added.
*
* @return A {@link LegacyRetryStrategy} with AWS-specific conditions added.
*/
public static LegacyRetryStrategy legacyRetryStrategy() {
LegacyRetryStrategy.Builder builder = SdkDefaultRetryStrategy.legacyRetryStrategyBuilder();
return configure(builder)
.build();
}

/**
* Returns an {@link AdaptiveRetryStrategy} with AWS-specific conditions added.
*
* @return An {@link AdaptiveRetryStrategy} with AWS-specific conditions added.
*/
public static AdaptiveRetryStrategy adaptiveRetryStrategy() {
AdaptiveRetryStrategy.Builder builder = SdkDefaultRetryStrategy.adaptiveRetryStrategyBuilder();
return configure(builder)
.build();
}

/**
* Configures a retry strategy using its builder to add AWS-specific retry exceptions.
*
* @param builder The builder to add the AWS-specific retry exceptions
* @return The given builder
* @param <T> The type of the builder extending {@link RetryStrategy.Builder}
*/
public static <T extends RetryStrategy.Builder<T, ?>> T configure(T builder) {
return builder.retryOnException(AwsRetryStrategy::retryOnAwsRetryableErrors);
}

private static boolean retryOnAwsRetryableErrors(Throwable ex) {
if (ex instanceof AwsServiceException) {
AwsServiceException exception = (AwsServiceException) ex;
return AwsErrorCode.RETRYABLE_ERROR_CODES.contains(exception.awsErrorDetails().errorCode());
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,15 @@ public final class DefaultRetryStrategy {
private DefaultRetryStrategy() {
}

/**
* Creates a non-retrying strategy.
*/
public static StandardRetryStrategy none() {
return standardStrategyBuilder()
.maxAttempts(1)
.build();
}

/**
* Create a new builder for a {@link StandardRetryStrategy}.
*
Expand Down
10 changes: 10 additions & 0 deletions core/sdk-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@
<artifactId>profiles</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>retries-api</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>retries</artifactId>
<version>${awsjavasdk.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import static software.amazon.awssdk.core.client.config.SdkClientOption.PROFILE_FILE_SUPPLIER;
import static software.amazon.awssdk.core.client.config.SdkClientOption.PROFILE_NAME;
import static software.amazon.awssdk.core.client.config.SdkClientOption.RETRY_POLICY;
import static software.amazon.awssdk.core.client.config.SdkClientOption.RETRY_STRATEGY;
import static software.amazon.awssdk.core.client.config.SdkClientOption.SCHEDULED_EXECUTOR_SERVICE;
import static software.amazon.awssdk.core.client.config.SdkClientOption.SIGNER_OVERRIDDEN;
import static software.amazon.awssdk.core.client.config.SdkClientOption.SYNC_HTTP_CLIENT;
Expand Down Expand Up @@ -77,6 +78,7 @@
import software.amazon.awssdk.core.internal.interceptor.HttpChecksumRequiredInterceptor;
import software.amazon.awssdk.core.internal.interceptor.HttpChecksumValidationInterceptor;
import software.amazon.awssdk.core.internal.interceptor.SyncHttpChecksumInTrailerInterceptor;
import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy;
import software.amazon.awssdk.core.retry.RetryMode;
import software.amazon.awssdk.core.retry.RetryPolicy;
import software.amazon.awssdk.core.util.SdkUserAgent;
Expand All @@ -89,6 +91,10 @@
import software.amazon.awssdk.profiles.ProfileFile;
import software.amazon.awssdk.profiles.ProfileFileSupplier;
import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
import software.amazon.awssdk.retries.AdaptiveRetryStrategy;
import software.amazon.awssdk.retries.LegacyRetryStrategy;
import software.amazon.awssdk.retries.StandardRetryStrategy;
import software.amazon.awssdk.retries.api.RetryStrategy;
import software.amazon.awssdk.utils.AttributeMap;
import software.amazon.awssdk.utils.Either;
import software.amazon.awssdk.utils.ScheduledExecutorUtils;
Expand Down Expand Up @@ -226,6 +232,7 @@ private SdkClientConfiguration setOverrides(SdkClientConfiguration configuration
builder.option(SCHEDULED_EXECUTOR_SERVICE, clientOverrideConfiguration.scheduledExecutorService().orElse(null));
builder.option(EXECUTION_INTERCEPTORS, clientOverrideConfiguration.executionInterceptors());
builder.option(RETRY_POLICY, clientOverrideConfiguration.retryPolicy().orElse(null));
builder.option(RETRY_STRATEGY, clientOverrideConfiguration.retryStrategy().orElse(null));
builder.option(ADDITIONAL_HTTP_HEADERS, clientOverrideConfiguration.headers());
builder.option(SIGNER, clientOverrideConfiguration.advancedOption(SIGNER).orElse(null));
builder.option(USER_AGENT_SUFFIX, clientOverrideConfiguration.advancedOption(USER_AGENT_SUFFIX).orElse(null));
Expand Down Expand Up @@ -314,21 +321,40 @@ private SdkClientConfiguration finalizeAsyncConfiguration(SdkClientConfiguration
*/
private SdkClientConfiguration finalizeConfiguration(SdkClientConfiguration config) {
RetryPolicy retryPolicy = resolveRetryPolicy(config);
RetryStrategy<?, ?> retryStrategy = resolveRetryStrategy(config);
String retryMode = resolveRetryMode(retryPolicy, retryStrategy);
return config.toBuilder()
.option(SCHEDULED_EXECUTOR_SERVICE, resolveScheduledExecutorService(config))
.option(EXECUTION_INTERCEPTORS, resolveExecutionInterceptors(config))
.option(RETRY_POLICY, retryPolicy)
.option(CLIENT_USER_AGENT, resolveClientUserAgent(config, retryPolicy))
.option(RETRY_STRATEGY, retryStrategy)
.option(CLIENT_USER_AGENT, resolveClientUserAgent(config, retryMode))
.build();
}

private String resolveClientUserAgent(SdkClientConfiguration config, RetryPolicy retryPolicy) {
private String resolveRetryMode(RetryPolicy retryPolicy, RetryStrategy<?, ?> retryStrategy) {
if (retryPolicy != null) {
return retryPolicy.retryMode().toString();
}
if (retryStrategy instanceof StandardRetryStrategy) {
return RetryMode.STANDARD.toString();
}
if (retryStrategy instanceof LegacyRetryStrategy) {
return RetryMode.LEGACY.toString();
}
if (retryStrategy instanceof AdaptiveRetryStrategy) {
return RetryMode.ADAPTIVE.toString();
}
return "UnknownRetryMode";
}

private String resolveClientUserAgent(SdkClientConfiguration config, String retryMode) {
return ApplyUserAgentStage.resolveClientUserAgent(config.option(USER_AGENT_PREFIX),
config.option(INTERNAL_USER_AGENT),
config.option(CLIENT_TYPE),
config.option(SYNC_HTTP_CLIENT),
config.option(ASYNC_HTTP_CLIENT),
retryPolicy);
retryMode);
}

private RetryPolicy resolveRetryPolicy(SdkClientConfiguration config) {
Expand All @@ -343,6 +369,25 @@ private RetryPolicy resolveRetryPolicy(SdkClientConfiguration config) {
.defaultRetryMode(config.option(SdkClientOption.DEFAULT_RETRY_MODE))
.resolve();
return RetryPolicy.forRetryMode(retryMode);
// TODO: fixme This will be changed like this to pick the configured retry strategy
// if no retry policy is configured.
/*
return config.option(SdkClientOption.RETRY_POLICY);
*/
}

private RetryStrategy<?, ?> resolveRetryStrategy(SdkClientConfiguration config) {
RetryStrategy<?, ?> strategy = config.option(RETRY_STRATEGY);
if (strategy != null) {
return strategy;
}

RetryMode retryMode = RetryMode.resolver()
.profileFile(config.option(SdkClientOption.PROFILE_FILE_SUPPLIER))
.profileName(config.option(SdkClientOption.PROFILE_NAME))
.defaultRetryMode(config.option(SdkClientOption.DEFAULT_RETRY_MODE))
.resolve();
return SdkDefaultRetryStrategy.forRetryMode(retryMode);
}

/**
Expand Down
Loading