S3 Async Client - Multipart download (#5164)

Multipart Download for S3 Async Client release

Author: L-Applin

.changes/next-release/feature-S3TransferManager-a02ba8b.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "S3 Transfer Manager",
+    "contributor": "",
+    "description": "This change enables multipart download for S3 Transfer Manager with the java-based Multipart S3 Async Client."
+}

core/sdk-core/src/main/java/software/amazon/awssdk/core/FileTransformerConfiguration.java

@@ -23,6 +23,7 @@
 import java.util.concurrent.ExecutorService;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.utils.ToString;
 import software.amazon.awssdk.utils.Validate;
 import software.amazon.awssdk.utils.builder.CopyableBuilder;
 import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
@@ -41,11 +42,19 @@ public final class FileTransformerConfiguration implements ToCopyableBuilder<Fil
     private final FileWriteOption fileWriteOption;
     private final FailureBehavior failureBehavior;
     private final ExecutorService executorService;
+    private final Long position;
 
     private FileTransformerConfiguration(DefaultBuilder builder) {
         this.fileWriteOption = Validate.paramNotNull(builder.fileWriteOption, "fileWriteOption");
         this.failureBehavior = Validate.paramNotNull(builder.failureBehavior, "failureBehavior");
         this.executorService = builder.executorService;
+        this.position = Validate.isNotNegativeOrNull(builder.position, "position");
+        if (fileWriteOption != FileWriteOption.WRITE_TO_POSITION && position != null) {
+            throw new IllegalArgumentException(String.format(
+                "'position' can only be used with 'WRITE_TO_POSITION' file write option, but was used with '%s'",
+                fileWriteOption
+            ));
+        }
     }
 
     /**
@@ -72,6 +81,18 @@ public Optional<ExecutorService> executorService() {
         return Optional.ofNullable(executorService);
     }
 
+    /**
+     * Exclusively used with {@link FileWriteOption#WRITE_TO_POSITION}. Configures the position, where to start writing to the
+     * existing file. The location correspond to the first byte where new data will be written. For example, if {@code 128} is
+     * configured, bytes 0-127 of the existing file will remain untouched and data will be written starting at byte 128. If not
+     * specified, defaults to 0.
+     *
+     * @return The offset at which to start overwriting data in the file.
+     */
+    public Long position() {
+        return position;
+    }
+
     /**
      * Create a {@link Builder}, used to create a {@link FileTransformerConfiguration}.
      */
@@ -137,6 +158,9 @@ public boolean equals(Object o) {
         if (failureBehavior != that.failureBehavior) {
             return false;
         }
+        if (!Objects.equals(position, that.position)) {
+            return false;
+        }
         return Objects.equals(executorService, that.executorService);
     }
 
@@ -145,6 +169,7 @@ public int hashCode() {
         int result = fileWriteOption != null ? fileWriteOption.hashCode() : 0;
         result = 31 * result + (failureBehavior != null ? failureBehavior.hashCode() : 0);
         result = 31 * result + (executorService != null ? executorService.hashCode() : 0);
+        result = 31 * result + (position != null ? position.hashCode() : 0);
         return result;
     }
 
@@ -165,7 +190,15 @@ public enum FileWriteOption {
         /**
          * Create a new file if it doesn't exist, otherwise append to the existing file.
          */
-        CREATE_OR_APPEND_TO_EXISTING
+        CREATE_OR_APPEND_TO_EXISTING,
+
+        /**
+         * Write to an existing file at the specified position, defined by the {@link FileTransformerConfiguration#position()}. If
+         * the file does not exist, a {@link java.nio.file.NoSuchFileException} will be thrown. If
+         * {@link FileTransformerConfiguration#position()} is not configured, start overwriting data at the beginning of the file
+         * (byte 0).
+         */
+        WRITE_TO_POSITION
     }
 
     /**
@@ -209,12 +242,24 @@ public interface Builder extends CopyableBuilder<Builder, FileTransformerConfigu
          * @return This object for method chaining.
          */
         Builder executorService(ExecutorService executorService);
+
+        /**
+         * Exclusively used with {@link FileWriteOption#WRITE_TO_POSITION}. Configures the position, where to start writing to the
+         * existing file. The location correspond to the first byte where new data will be written. For example, if {@code 128} is
+         * configured, bytes 0-127 of the existing file will remain untouched and data will be written starting at byte 128. If
+         * not specified, defaults to 0.
+         *
+         * @param writePosition the position at where to start writing data to the file.
+         * @return This object for method chaining.
+         */
+        Builder position(Long writePosition);
     }
 
     private static final class DefaultBuilder implements Builder {
         private FileWriteOption fileWriteOption;
         private FailureBehavior failureBehavior;
         private ExecutorService executorService;
+        private Long position;
 
         private DefaultBuilder() {
         }
@@ -223,6 +268,7 @@ private DefaultBuilder(FileTransformerConfiguration fileTransformerConfiguration
             this.fileWriteOption = fileTransformerConfiguration.fileWriteOption;
             this.failureBehavior = fileTransformerConfiguration.failureBehavior;
             this.executorService = fileTransformerConfiguration.executorService;
+            this.position = fileTransformerConfiguration.position;
         }
 
         @Override
@@ -243,10 +289,25 @@ public Builder executorService(ExecutorService executorService) {
             return this;
         }
 
+        @Override
+        public Builder position(Long position) {
+            this.position = position;
+            return this;
+        }
+
         @Override
         public FileTransformerConfiguration build() {
             return new FileTransformerConfiguration(this);
         }
     }
 
-}
+    @Override
+    public String toString() {
+        return ToString.builder("FileTransformerConfiguration")
+                       .add("fileWriteOption", this.fileWriteOption)
+                       .add("failureBehavior", this.failureBehavior)
+                       .add("executorService", this.executorService)
+                       .add("position", this.position)
+                       .build();
+    }
+}

core/sdk-core/src/main/java/software/amazon/awssdk/core/SplittingTransformerConfiguration.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.core;
+
+import java.util.Objects;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.core.internal.async.SplittingTransformer;
+import software.amazon.awssdk.utils.ToString;
+import software.amazon.awssdk.utils.Validate;
+import software.amazon.awssdk.utils.builder.CopyableBuilder;
+import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
+
+/**
+ * Configuration options for {@link AsyncResponseTransformer#split(SplittingTransformerConfiguration)} to configure how the SDK
+ * should split the {@link AsyncResponseTransformer}.
+ *
+ * @see #builder()
+ */
+@SdkPublicApi
+public final class SplittingTransformerConfiguration implements ToCopyableBuilder<SplittingTransformerConfiguration.Builder,
+    SplittingTransformerConfiguration> {
+
+    private final Long bufferSizeInBytes;
+
+    private SplittingTransformerConfiguration(DefaultBuilder builder) {
+        this.bufferSizeInBytes = Validate.paramNotNull(builder.bufferSize, "bufferSize");
+    }
+
+    /**
+     * Create a {@link Builder}, used to create a {@link SplittingTransformerConfiguration}.
+     */
+    public static Builder builder() {
+        return new DefaultBuilder();
+    }
+
+    /**
+     * @return the buffer size
+     */
+    public Long bufferSizeInBytes() {
+        return bufferSizeInBytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        SplittingTransformerConfiguration that = (SplittingTransformerConfiguration) o;
+
+        return Objects.equals(bufferSizeInBytes, that.bufferSizeInBytes);
+    }
+
+    @Override
+    public int hashCode() {
+        return bufferSizeInBytes != null ? bufferSizeInBytes.hashCode() : 0;
+    }
+
+    @Override
+    public String toString() {
+        return ToString.builder("SplittingTransformerConfiguration")
+            .add("bufferSizeInBytes", bufferSizeInBytes)
+            .build();
+    }
+
+    @Override
+    public Builder toBuilder() {
+        return new DefaultBuilder(this);
+    }
+
+    public interface Builder extends CopyableBuilder<Builder, SplittingTransformerConfiguration> {
+
+        /**
+         * Configures the maximum amount of memory in bytes buffered by the {@link SplittingTransformer}.
+         *
+         * @param bufferSize the buffer size in bytes
+         * @return This object for method chaining.
+         */
+        Builder bufferSizeInBytes(Long bufferSize);
+    }
+
+    private static final class DefaultBuilder implements Builder {
+        private Long bufferSize;
+
+        private DefaultBuilder(SplittingTransformerConfiguration configuration) {
+            this.bufferSize = configuration.bufferSizeInBytes;
+        }
+
+        private DefaultBuilder() {
+        }
+
+        @Override
+        public Builder bufferSizeInBytes(Long bufferSize) {
+            this.bufferSize = bufferSize;
+            return this;
+        }
+
+        @Override
+        public SplittingTransformerConfiguration build() {
+            return new SplittingTransformerConfiguration(this);
+        }
+    }
+}