Added a new dualstackEnabled property to every client builder. (#2818)

This option can be used to make calls be invoked against AWS endpoints which return IPv6 records. This can also be enabled via the `AWS_USE_DUALSTACK_ENDPOINT` environment variable, `aws.useDualstackEndpoint` system property, or the `use_dualstack_endpoint` profile file property:

1. This PR builds on top of the 'endpoint variants' #2808, by specifying the DUALSTACK endpoint variant whenever the customer requests dualstack to be enabled.
2. The bulk of the files in this PR are addressing the fact that we already have service configuration for enabling dualstack for S3 and S3 control. These existing options have been deprecated in favor of the new option that works for all services.

Author: millems

.changes/next-release/deprecation-AmazonS3-21b2ad3.json

@@ -0,0 +1,6 @@
+{
+    "category": "Amazon S3", 
+    "contributor": "", 
+    "type": "deprecation", 
+    "description": "Deprecated `S3Configuration.Builder`'s `dualstackEnabled` in favor of the new service-standard `S3ClientBuilder.dualstackEnabled`."
+}

.changes/next-release/deprecation-AmazonS3Control-b22c29a.json

@@ -0,0 +1,6 @@
+{
+    "category": "Amazon S3 Control", 
+    "contributor": "", 
+    "type": "deprecation", 
+    "description": "Deprecated `S3ControlConfiguration.Builder`'s `dualstackEnabled` in favor of the new service-standard `S3ControlClientBuilder.dualstackEnabled`."
+}

.changes/next-release/feature-AWSSDKforJava-6255065.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Added a new `dualstackEnabled` property to every client builder, which can be used to make calls be invoked against AWS endpoints which return IPv6 records. This can also be enabled via the `AWS_USE_DUALSTACK_ENDPOINT` environment variable, `aws.useDualstackEndpoint` system property, or the `use_dualstack_endpoint` profile file property."
+}

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java

@@ -42,6 +42,12 @@ public class CustomizationConfig {
      * does not have advanced configuration.
      */
     private String serviceSpecificClientConfigClass;
+
+    /**
+     * Whether a service has a dualstack configuration in its {@link #serviceSpecificClientConfigClass}.
+     */
+    private boolean serviceConfigHasDualstackConfig = false;
+
     /**
      * Specify shapes to be renamed.
      */
@@ -197,6 +203,8 @@ public class CustomizationConfig {
 
     private RetryMode defaultRetryMode;
 
+
+
     private CustomizationConfig() {
     }
 
@@ -244,6 +252,14 @@ public void setServiceSpecificClientConfigClass(String serviceSpecificClientConf
         this.serviceSpecificClientConfigClass = serviceSpecificClientConfig;
     }
 
+    public boolean getServiceConfigHasDualstackConfig() {
+        return serviceConfigHasDualstackConfig;
+    }
+
+    public void setServiceConfigHasDualstackConfig(boolean serviceConfigHasDualstackConfig) {
+        this.serviceConfigHasDualstackConfig = serviceConfigHasDualstackConfig;
+    }
+
     public List<ConvenienceTypeOverload> getConvenienceTypeOverloads() {
         return this.convenienceTypeOverloads;
     }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/BaseClientBuilderClass.java

@@ -34,6 +34,7 @@
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.auth.signer.Aws4Signer;
 import software.amazon.awssdk.awscore.client.builder.AwsDefaultClientBuilder;
+import software.amazon.awssdk.awscore.client.config.AwsClientOption;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
@@ -54,6 +55,7 @@
 import software.amazon.awssdk.utils.AttributeMap;
 import software.amazon.awssdk.utils.CollectionUtils;
 import software.amazon.awssdk.utils.StringUtils;
+import software.amazon.awssdk.utils.Validate;
 
 public class BaseClientBuilderClass implements ClassSpec {
     private final IntermediateModel model;
@@ -241,9 +243,20 @@ private MethodSpec finalizeServiceConfigurationMethod() {
         if (StringUtils.isNotBlank(clientConfigClassName)) {
             ClassName clientConfigClass = ClassName.bestGuess(clientConfigClassName);
             builder.addCode("$1T.Builder c = (($1T) config.option($2T.SERVICE_CONFIGURATION)).toBuilder();" +
-                            "c.profileFile(c.profileFile() != null ? c.profileFile() : config.option($2T.PROFILE_FILE))" +
-                            " .profileName(c.profileName() != null ? c.profileName() : config.option($2T.PROFILE_NAME));",
+                            "c.profileFile(c.profileFile() != null ? c.profileFile() : config.option($2T.PROFILE_FILE));" +
+                            "c.profileName(c.profileName() != null ? c.profileName() : config.option($2T.PROFILE_NAME));",
                             clientConfigClass, SdkClientOption.class);
+
+            if (model.getCustomizationConfig().getServiceConfigHasDualstackConfig()) {
+                builder.addCode("if (c.dualstackEnabled() != null) {")
+                       .addCode("    $T.validState(config.option($T.DUALSTACK_ENDPOINT_ENABLED) == null, \"Dualstack has been "
+                                + "configured on both $L and the client/global level. Please limit dualstack configuration to "
+                                + "one location.\");",
+                                Validate.class, AwsClientOption.class, clientConfigClassName)
+                       .addCode("} else {")
+                       .addCode("    c.dualstackEnabled(config.option($T.DUALSTACK_ENDPOINT_ENABLED));", AwsClientOption.class)
+                       .addCode("}");
+            }
         }
 
         // Update configuration

codegen/src/test/resources/software/amazon/awssdk/codegen/poet/builder/test-client-builder-class.java

@@ -7,6 +7,7 @@
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.auth.signer.Aws4Signer;
 import software.amazon.awssdk.awscore.client.builder.AwsDefaultClientBuilder;
+import software.amazon.awssdk.awscore.client.config.AwsClientOption;
 import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
 import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
 import software.amazon.awssdk.core.client.config.SdkClientOption;
@@ -15,6 +16,7 @@
 import software.amazon.awssdk.core.signer.Signer;
 import software.amazon.awssdk.utils.AttributeMap;
 import software.amazon.awssdk.utils.CollectionUtils;
+import software.amazon.awssdk.utils.Validate;
 
 /**
  * Internal base class for {@link DefaultJsonClientBuilder} and {@link DefaultJsonAsyncClientBuilder}.
@@ -47,8 +49,15 @@ protected final SdkClientConfiguration finalizeServiceConfiguration(SdkClientCon
         interceptors = CollectionUtils.mergeLists(interceptors, config.option(SdkClientOption.EXECUTION_INTERCEPTORS));
         ServiceConfiguration.Builder c = ((ServiceConfiguration) config.option(SdkClientOption.SERVICE_CONFIGURATION))
             .toBuilder();
-        c.profileFile(c.profileFile() != null ? c.profileFile() : config.option(SdkClientOption.PROFILE_FILE)).profileName(
-            c.profileName() != null ? c.profileName() : config.option(SdkClientOption.PROFILE_NAME));
+        c.profileFile(c.profileFile() != null ? c.profileFile() : config.option(SdkClientOption.PROFILE_FILE));
+        c.profileName(c.profileName() != null ? c.profileName() : config.option(SdkClientOption.PROFILE_NAME));
+        if (c.dualstackEnabled() != null) {
+            Validate.validState(
+                config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED) == null,
+                "Dualstack has been configured on both ServiceConfiguration and the client/global level. Please limit dualstack configuration to one location.");
+        } else {
+            c.dualstackEnabled(config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED));
+        }
         return config.toBuilder().option(SdkClientOption.EXECUTION_INTERCEPTORS, interceptors)
                      .option(SdkClientOption.RETRY_POLICY, MyServiceRetryPolicy.resolveRetryPolicy(config))
                      .option(SdkClientOption.SERVICE_CONFIGURATION, c.build()).build();

codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/json/customization.config

@@ -5,6 +5,7 @@
     "presignersFqcn": "software.amazon.awssdk.services.acm.presign.AcmClientPresigners",
     "serviceSpecificHttpConfig": "software.amazon.MyServiceHttpConfig",
     "serviceSpecificClientConfigClass": "ServiceConfiguration",
+    "serviceConfigHasDualstackConfig": true,
     "customRetryPolicy": "software.amazon.MyServiceRetryPolicy",
     "verifiedSimpleMethods" : ["paginatedOperationWithResultKey"],
     "blacklistedSimpleMethods" : [

codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/rest-json/customization.config

@@ -5,6 +5,7 @@
     "presignersFqcn": "software.amazon.awssdk.services.acm.presign.AcmClientPresigners",
     "serviceSpecificHttpConfig": "software.amazon.MyServiceHttpConfig",
     "serviceSpecificClientConfigClass": "ServiceConfiguration",
+    "serviceConfigHasDualstackConfig": true,
     "customRetryPolicy": "software.amazon.MyServiceRetryPolicy",
     "verifiedSimpleMethods" : ["paginatedOperationWithResultKey"],
     "blacklistedSimpleMethods" : [

core/aws-core/src/main/java/software/amazon/awssdk/awscore/AwsExecutionAttribute.java

@@ -39,6 +39,9 @@ public final class AwsExecutionAttribute extends SdkExecutionAttribute {
      */
     public static final ExecutionAttribute<String> ENDPOINT_PREFIX = new ExecutionAttribute<>("AwsEndpointPrefix");
 
+    public static final ExecutionAttribute<Boolean> DUALSTACK_ENDPOINT_ENABLED =
+        new ExecutionAttribute<>("DualstackEndpointsEnabled");
+
     private AwsExecutionAttribute() {
     }
 }

core/aws-core/src/main/java/software/amazon/awssdk/awscore/client/builder/AwsClientBuilder.java

@@ -51,7 +51,6 @@ public interface AwsClientBuilder<BuilderT extends AwsClientBuilder<BuilderT, Cl
      */
     BuilderT credentialsProvider(AwsCredentialsProvider credentialsProvider);
 
-
     /**
      * Configure the region with which the SDK should communicate.
      *
@@ -62,6 +61,24 @@ public interface AwsClientBuilder<BuilderT extends AwsClientBuilder<BuilderT, Cl
      *     <li>Check the {user.home}/.aws/credentials and {user.home}/.aws/config files for the region.</li>
      *     <li>If running in EC2, check the EC2 metadata service for the region.</li>
      * </ol>
+     *
+     * <p>If the region is not found in any of the locations above, an exception will be thrown at {@link #build()} time.
      */
     BuilderT region(Region region);
+
+    /**
+     * Configure whether the SDK should use the AWS dualstack endpoint.
+     *
+     * <p>If this is not specified, the SDK will attempt to determine whether the dualstack endpoint should be used
+     * automatically using the following logic:
+     * <ol>
+     *     <li>Check the 'aws.useDualstackEndpoint' system property for 'true' or 'false'.</li>
+     *     <li>Check the 'AWS_USE_DUALSTACK_ENDPOINT' environment variable for 'true' or 'false'.</li>
+     *     <li>Check the {user.home}/.aws/credentials and {user.home}/.aws/config files for the 'use_dualstack_endpoint'
+     *     property set to 'true' or 'false'.</li>
+     * </ol>
+     *
+     * <p>If the setting is not found in any of the locations above, 'false' will be used.
+     */
+    BuilderT dualstackEnabled(Boolean dualstackEndpointEnabled);
 }

core/aws-core/src/main/java/software/amazon/awssdk/awscore/client/builder/AwsDefaultClientBuilder.java

@@ -26,6 +26,7 @@
 import software.amazon.awssdk.awscore.client.config.AwsAdvancedClientOption;
 import software.amazon.awssdk.awscore.client.config.AwsClientOption;
 import software.amazon.awssdk.awscore.endpoint.DefaultServiceEndpointBuilder;
+import software.amazon.awssdk.awscore.endpoint.DualstackEnabledProvider;
 import software.amazon.awssdk.awscore.eventstream.EventStreamInitialRequestInterceptor;
 import software.amazon.awssdk.awscore.interceptor.HelpfulUnknownHostExceptionInterceptor;
 import software.amazon.awssdk.awscore.retry.AwsRetryPolicy;
@@ -141,6 +142,8 @@ protected final SdkClientConfiguration finalizeChildConfiguration(SdkClientConfi
 
         configuration = configuration.toBuilder()
                                      .option(AwsClientOption.AWS_REGION, resolveRegion(configuration))
+                                     .option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED,
+                                             resolveDualstackEndpointEnabled(configuration))
                                      .build();
 
         return configuration.toBuilder()
@@ -180,6 +183,7 @@ private URI endpointFromConfig(SdkClientConfiguration config) {
             .withRegion(config.option(AwsClientOption.AWS_REGION))
             .withProfileFile(config.option(SdkClientOption.PROFILE_FILE))
             .withProfileName(config.option(SdkClientOption.PROFILE_NAME))
+            .withDualstackEnabled(config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED))
             .getServiceEndpoint();
     }
 
@@ -210,6 +214,29 @@ private Region regionFromDefaultProvider(SdkClientConfiguration config) {
                                             .getRegion();
     }
 
+    /**
+     * Resolve whether a dualstack endpoint should be used for this client.
+     */
+    private Boolean resolveDualstackEndpointEnabled(SdkClientConfiguration config) {
+        return config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED) != null
+               ? config.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED)
+               : dualstackEndpointFromDefaultProvider(config);
+    }
+
+    /**
+     * Load the dualstack endpoint setting from the default provider logic.
+     */
+    private Boolean dualstackEndpointFromDefaultProvider(SdkClientConfiguration config) {
+        ProfileFile profileFile = config.option(SdkClientOption.PROFILE_FILE);
+        String profileName = config.option(SdkClientOption.PROFILE_NAME);
+        return DualstackEnabledProvider.builder()
+                                       .profileFile(() -> profileFile)
+                                       .profileName(profileName)
+                                       .build()
+                                       .isDualstackEnabled()
+                                       .orElse(null);
+    }
+
     /**
      * Resolve the credentials that should be used based on the customer's configuration.
      */
@@ -251,6 +278,16 @@ public final void setRegion(Region region) {
         region(region);
     }
 
+    @Override
+    public BuilderT dualstackEnabled(Boolean dualstackEndpointEnabled) {
+        clientConfiguration.option(AwsClientOption.DUALSTACK_ENDPOINT_ENABLED, dualstackEndpointEnabled);
+        return thisBuilder();
+    }
+
+    public final void setDualstackEnabled(Boolean dualstackEndpointEnabled) {
+        dualstackEnabled(dualstackEndpointEnabled);
+    }
+
     @Override
     public final BuilderT credentialsProvider(AwsCredentialsProvider credentialsProvider) {
         clientConfiguration.option(AwsClientOption.CREDENTIALS_PROVIDER, credentialsProvider);

core/aws-core/src/main/java/software/amazon/awssdk/awscore/client/config/AwsClientOption.java

@@ -40,6 +40,12 @@ public final class AwsClientOption<T> extends ClientOption<T> {
      */
     public static final AwsClientOption<Region> SIGNING_REGION = new AwsClientOption<>(Region.class);
 
+    /**
+     * Whether the SDK should resolve dualstack endpoints instead of default endpoints. See
+     * {@link AwsClientBuilder#dualstackEnabled(Boolean)}.
+     */
+    public static final AwsClientOption<Boolean> DUALSTACK_ENDPOINT_ENABLED = new AwsClientOption<>(Boolean.class);
+
     /**
      * Scope name to use during signing of a request.
      */

core/aws-core/src/main/java/software/amazon/awssdk/awscore/endpoint/DefaultServiceEndpointBuilder.java

@@ -17,12 +17,16 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.List;
 import software.amazon.awssdk.annotations.NotThreadSafe;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.profiles.ProfileFile;
+import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
+import software.amazon.awssdk.regions.EndpointTag;
 import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.regions.ServiceEndpointKey;
 import software.amazon.awssdk.regions.ServiceMetadata;
 import software.amazon.awssdk.utils.Validate;
 
@@ -40,6 +44,7 @@ public final class DefaultServiceEndpointBuilder {
     private Region region;
     private ProfileFile profileFile;
     private String profileName;
+    private Boolean dualstackEnabled;
 
     public DefaultServiceEndpointBuilder(String serviceName, String protocol) {
         this.serviceName = Validate.paramNotNull(serviceName, "serviceName");
@@ -64,18 +69,49 @@ public DefaultServiceEndpointBuilder withProfileName(String profileName) {
         return this;
     }
 
+    public DefaultServiceEndpointBuilder withDualstackEnabled(Boolean dualstackEnabled) {
+        this.dualstackEnabled = dualstackEnabled;
+        return this;
+    }
+
     public URI getServiceEndpoint() {
+        if (profileFile == null) {
+            profileFile = ProfileFile.defaultProfileFile();
+        }
+
+        if (profileName == null) {
+            profileName = ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow();
+        }
+
+        if (dualstackEnabled == null) {
+            dualstackEnabled = DualstackEnabledProvider.builder()
+                                                       .profileFile(() -> profileFile)
+                                                       .profileName(profileName)
+                                                       .build()
+                                                       .isDualstackEnabled()
+                                                       .orElse(false);
+        }
+
+        List<EndpointTag> endpointTags = new ArrayList<>();
+        if (dualstackEnabled) {
+            endpointTags.add(EndpointTag.DUALSTACK);
+        }
+
         ServiceMetadata serviceMetadata = ServiceMetadata.of(serviceName)
                                                          .reconfigure(c -> c.profileFile(() -> profileFile)
                                                                             .profileName(profileName));
-        URI endpoint = addProtocolToServiceEndpoint(serviceMetadata.endpointFor(region));
+        URI endpoint = addProtocolToServiceEndpoint(serviceMetadata.endpointFor(ServiceEndpointKey.builder()
+                                                                                                  .region(region)
+                                                                                                  .tags(endpointTags)
+                                                                                                  .build()));
 
         if (endpoint.getHost() == null) {
-            String error = "Configured region (" + region + ") resulted in an invalid URI: " + endpoint;
+            String error = "Configured region (" + region + ") and tags (" + endpointTags + ") resulted in an invalid URI: "
+                           + endpoint + ". This is usually caused by an invalid region configuration.";
 
             List<Region> exampleRegions = serviceMetadata.regions();
             if (!exampleRegions.isEmpty()) {
-                error += " Valid region examples: " + exampleRegions;
+                error += " Valid regions: " + exampleRegions;
             }
 
             throw SdkClientException.create(error);