S3 Multipart API implementation

.changes/next-release/feature-AWSSDKforJavav2-5d806ad.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "StephenFlavin",
+    "description": "Add \"unsafe\" and \"fromRemaining\" AsyncRequestBody constructors for byte arrays and ByteBuffers"
+}

.changes/next-release/feature-AWSSDKforJavav2-70eb163.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Cross region bucket access for S3 Client. This feature will allow users to access buckets of different region using a single cross region configured client."
+}

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java

@@ -227,6 +227,18 @@ public class CustomizationConfig {
      */
     private String asyncClientDecorator;
 
+    /**
+     * For S3AsyncClient only. Fully qualified name of the class to be used for multipart configuration
+     */
+    private String multipartConfigurationClass;
+
+    /**
+     * For S3AsyncClient only. Javadoc associated with the generated method on the S3AsyncClientBuilder interface for multipart
+     * configuration.
+     */
+    private String multipartMethodJavadoc;
+
+
     /**
      * Whether to skip generating endpoint tests from endpoint-tests.json
      */
@@ -665,4 +677,21 @@ public Map<String, ClientContextParam> getCustomClientContextParams() {
     public void setCustomClientContextParams(Map<String, ClientContextParam> customClientContextParams) {
         this.customClientContextParams = customClientContextParams;
     }
+
+    public String getMultipartConfigurationClass() {
+        return this.multipartConfigurationClass;
+    }
+
+    public void setMultipartConfigurationClass(String multipartConfigurationClass) {
+        this.multipartConfigurationClass = multipartConfigurationClass;
+    }
+
+    public String getMultipartMethodDoc() {
+        return this.multipartMethodJavadoc;
+    }
+
+    public void setMultipartMethodDoc(String multipartMethodJavadoc) {
+        this.multipartMethodJavadoc = multipartMethodJavadoc;
+    }
+
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java

@@ -15,6 +15,8 @@
 
 package software.amazon.awssdk.codegen.model.config.customization;
 
+import software.amazon.awssdk.utils.ToString;
+
 public class ServiceConfig {
     /**
      * Specifies the name of the client configuration class to use if a service
@@ -112,4 +114,18 @@ public boolean hasAccelerateModeEnabledProperty() {
     public void setHasAccelerateModeEnabledProperty(boolean hasAccelerateModeEnabledProperty) {
         this.hasAccelerateModeEnabledProperty = hasAccelerateModeEnabledProperty;
     }
+
+    @Override
+    public String toString() {
+        return ToString.builder("ServiceConfig")
+                       .add("className", className)
+                       .add("hasDualstackProperty", hasDualstackProperty)
+                       .add("hasFipsProperty", hasFipsProperty)
+                       .add("hasUseArnRegionProperty", hasUseArnRegionProperty)
+                       .add("hasMultiRegionEnabledProperty", hasMultiRegionEnabledProperty)
+                       .add("hasPathStyleAccessEnabledProperty", hasPathStyleAccessEnabledProperty)
+                       .add("hasAccelerateModeEnabledProperty", hasAccelerateModeEnabledProperty)
+                       .add("hasCrossRegionAccessEnabledProperty", hasCrossRegionAccessEnabledProperty)
+                       .build();
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java

@@ -20,7 +20,7 @@
 import java.util.Collections;
 
 /**
- * Represents the a Poet generated class
+ * Represents a Poet generated class
  */
 public interface ClassSpec {
 

codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java

@@ -17,6 +17,7 @@
 
 import com.squareup.javapoet.ClassName;
 import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.ParameterSpec;
 import com.squareup.javapoet.ParameterizedTypeName;
 import com.squareup.javapoet.TypeSpec;
 import java.net.URI;
@@ -33,6 +34,7 @@
 import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
 import software.amazon.awssdk.core.client.config.SdkClientOption;
 import software.amazon.awssdk.endpoints.EndpointProvider;
+import software.amazon.awssdk.utils.StringUtils;
 
 public class AsyncClientBuilderClass implements ClassSpec {
     private final IntermediateModel model;
@@ -59,12 +61,12 @@ public AsyncClientBuilderClass(IntermediateModel model) {
     @Override
     public TypeSpec poetSpec() {
         TypeSpec.Builder builder =
-                PoetUtils.createClassBuilder(builderClassName)
-                         .addAnnotation(SdkInternalApi.class)
-                         .addModifiers(Modifier.FINAL)
-                         .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName))
-                         .addSuperinterface(builderInterfaceName)
-                         .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName);
+            PoetUtils.createClassBuilder(builderClassName)
+                     .addAnnotation(SdkInternalApi.class)
+                     .addModifiers(Modifier.FINAL)
+                     .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName))
+                     .addSuperinterface(builderInterfaceName)
+                     .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName);
 
         if (model.getEndpointOperation().isPresent()) {
             builder.addMethod(endpointDiscoveryEnabled());
@@ -80,6 +82,11 @@ public TypeSpec poetSpec() {
             builder.addMethod(bearerTokenProviderMethod());
         }
 
+        boolean multipartEnabled = StringUtils.isNotBlank(model.getCustomizationConfig().getMultipartConfigurationClass());
+        if (multipartEnabled) {
+            builder.addMethod(multipartConfigMethod());
+        }
+
         builder.addMethod(buildClientMethod());
         builder.addMethod(initializeServiceClientConfigMethod());
 
@@ -124,15 +131,15 @@ private MethodSpec endpointProviderMethod() {
 
     private MethodSpec buildClientMethod() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("buildClient")
-                                         .addAnnotation(Override.class)
-                                         .addModifiers(Modifier.PROTECTED, Modifier.FINAL)
-                                         .returns(clientInterfaceName)
-                                         .addStatement("$T clientConfiguration = super.asyncClientConfiguration()",
-                                                       SdkClientConfiguration.class).addStatement("this.validateClientOptions"
-                                                                                                  + "(clientConfiguration)")
-                                         .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig"
-                                                       + "(clientConfiguration)",
-                                                       serviceConfigClassName);
+                                               .addAnnotation(Override.class)
+                                               .addModifiers(Modifier.PROTECTED, Modifier.FINAL)
+                                               .returns(clientInterfaceName)
+                                               .addStatement("$T clientConfiguration = super.asyncClientConfiguration()",
+                                                             SdkClientConfiguration.class)
+                                               .addStatement("this.validateClientOptions(clientConfiguration)")
+                                               .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig"
+                                                             + "(clientConfiguration)",
+                                                             serviceConfigClassName);
 
         builder.addStatement("$1T client = new $2T(serviceClientConfiguration, clientConfiguration)",
                              clientInterfaceName, clientClassName);
@@ -156,6 +163,20 @@ private MethodSpec bearerTokenProviderMethod() {
                          .build();
     }
 
+    private MethodSpec multipartConfigMethod() {
+        ClassName mulitpartConfigClassName =
+            PoetUtils.classNameFromFqcn(model.getCustomizationConfig().getMultipartConfigurationClass());
+        return MethodSpec.methodBuilder("multipartConfiguration")
+                         .addAnnotation(Override.class)
+                         .addModifiers(Modifier.PUBLIC)
+                         .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfig").build())
+                         .returns(builderClassName)
+                         .addStatement("clientContextParams.put($T.MULTIPART_CONFIGURATION_KEY, multipartConfig)",
+                                       mulitpartConfigClassName)
+                         .addStatement("return this")
+                         .build();
+    }
+
     private MethodSpec initializeServiceClientConfigMethod() {
         return MethodSpec.methodBuilder("initializeServiceClientConfig").addModifiers(Modifier.PRIVATE)
                          .addParameter(SdkClientConfiguration.class, "clientConfig")

codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java

@@ -17,34 +17,83 @@
 
 import com.squareup.javapoet.ClassName;
 import com.squareup.javapoet.CodeBlock;
+import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.ParameterSpec;
 import com.squareup.javapoet.ParameterizedTypeName;
 import com.squareup.javapoet.TypeSpec;
+import java.util.function.Consumer;
+import javax.lang.model.element.Modifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.awscore.client.builder.AwsAsyncClientBuilder;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.poet.ClassSpec;
 import software.amazon.awssdk.codegen.poet.PoetUtils;
+import software.amazon.awssdk.utils.StringUtils;
 
 public class AsyncClientBuilderInterface implements ClassSpec {
+    static Logger log = LoggerFactory.getLogger(AsyncClientBuilderInterface.class);
     private final ClassName builderInterfaceName;
     private final ClassName clientInterfaceName;
     private final ClassName baseBuilderInterfaceName;
+    private final IntermediateModel model;
 
     public AsyncClientBuilderInterface(IntermediateModel model) {
         String basePackage = model.getMetadata().getFullClientPackageName();
         this.clientInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncInterface());
         this.builderInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncBuilderInterface());
         this.baseBuilderInterfaceName = ClassName.get(basePackage, model.getMetadata().getBaseBuilderInterface());
+        this.model = model;
     }
 
     @Override
     public TypeSpec poetSpec() {
-        return PoetUtils.createInterfaceBuilder(builderInterfaceName)
-                        .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class),
-                                                                     builderInterfaceName, clientInterfaceName))
-                        .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName,
-                                                                     builderInterfaceName, clientInterfaceName))
-                        .addJavadoc(getJavadoc())
-                        .build();
+        TypeSpec.Builder builder = PoetUtils
+            .createInterfaceBuilder(builderInterfaceName)
+            .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class),
+                                                         builderInterfaceName, clientInterfaceName))
+            .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName,
+                                                         builderInterfaceName, clientInterfaceName))
+            .addJavadoc(getJavadoc());
+
+        boolean multipartEnabled = StringUtils.isNotBlank(model.getCustomizationConfig().getMultipartConfigurationClass());
+        if (multipartEnabled) {
+            includeMultipartMethod(builder);
+        }
+        return builder.build();
+    }
+
+    private void includeMultipartMethod(TypeSpec.Builder builder) {
+        log.debug("Adding multipart config methods to builder interface for service '{}'", model.getMetadata().getServiceId());
+        String multipartConfigClass = model.getCustomizationConfig().getMultipartConfigurationClass();
+        String multipartMethodJavaDoc = model.getCustomizationConfig().getMultipartMethodDoc();
+        ClassName mulitpartConfigClassName = PoetUtils.classNameFromFqcn(multipartConfigClass);
+        String multiPartConfigMethodName = "multipartConfiguration";
+        builder.addMethod(
+            MethodSpec.methodBuilder(multiPartConfigMethodName)
+                      .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC)
+                      .returns(builderInterfaceName)
+                      .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfiguration").build())
+                      .addCode("throw new $T();", UnsupportedOperationException.class)
+                      .addJavadoc(CodeBlock.of(multipartMethodJavaDoc))
+                      .build());
+
+        ClassName mulitpartConfigBuilderClassName = PoetUtils.classNameFromFqcn(multipartConfigClass + ".Builder");
+
+        ParameterizedTypeName consumerBuilderType = ParameterizedTypeName.get(ClassName.get(Consumer.class),
+                                                                              mulitpartConfigBuilderClassName);
+        builder.addMethod(
+            MethodSpec.methodBuilder(multiPartConfigMethodName)
+                      .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC)
+                      .returns(builderInterfaceName)
+                      .addParameter(ParameterSpec.builder(consumerBuilderType, "multipartConfiguration").build())
+                      .addStatement("$T builder = $T.builder()",
+                                    mulitpartConfigBuilderClassName,
+                                    mulitpartConfigClassName)
+                      .addStatement("multipartConfiguration.accept(builder)")
+                      .addStatement("return multipartConfiguration(builder.build())")
+                      .addJavadoc(CodeBlock.of(multipartMethodJavaDoc))
+                      .build());
     }
 
     @Override

services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrossRegionCrtIntegrationTest.java

@@ -31,7 +31,6 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadLocalRandom;
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;

services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java

@@ -31,7 +31,6 @@
 import javax.crypto.KeyGenerator;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -66,7 +65,7 @@ public static void setUp() throws Exception {
                                            .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
                                            .region(DEFAULT_REGION)
                                            .build();
-        s3MpuClient = new MultipartS3AsyncClient(s3Async);
+        s3MpuClient = new MultipartS3AsyncClient(s3Async, MultipartConfiguration.create());
     }
 
     @AfterAll

services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientBuilderTest.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.multipart;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient;
+
+class S3MultipartClientBuilderTest {
+
+    @Test
+    void builderWithMultipartConfig_shouldBuildMultipartClient() {
+        S3AsyncClient client = S3AsyncClient.builder()
+                                            .multipartConfiguration(MultipartConfiguration.builder().build())
+                                            .region(Region.US_EAST_1)
+                                            .build();
+        assertThat(client).isInstanceOf(MultipartS3AsyncClient.class);
+
+    }
+
+    @Test
+    void builderWithMultipartConfig_consumerBuilder_shouldBuildMultipartClient() {
+        S3AsyncClient client = S3AsyncClient.builder()
+                                            .multipartConfiguration(b -> b.multipartEnabled(true))
+                                            .region(Region.US_EAST_1)
+                                            .build();
+        assertThat(client).isInstanceOf(MultipartS3AsyncClient.class);
+    }
+
+    @Test
+    void builderWithMultipartConfig_disabled_shouldNotBuildMultipartClient() {
+        S3AsyncClient client = S3AsyncClient.builder()
+                                            .multipartConfiguration(b -> b.multipartEnabled(false))
+                                            .region(Region.US_EAST_1)
+                                            .build();
+        assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class);
+    }
+
+    @Test
+    void builderWithoutMultipartConfig_shouldNotBeMultipartClient() {
+        S3AsyncClient client = S3AsyncClient.builder()
+                                            .region(Region.US_EAST_1)
+                                            .build();
+        assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class);
+    }
+}