…Create nonexistent parent directories in AsyncResponseTransformer#toF

.changes/next-release/bugfix-AWSSDKforJavav2-5fdaf2b.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "bugfix", 
+    "description": "`AsyncResponseTransformer#toFile` and `ResponseTransformer#toFile` will now create all nonexistent parent directories if they don't exist instead of throwing NoSuchFileException`"
+}

core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncResponseTransformer.java

@@ -111,7 +111,7 @@ public interface AsyncResponseTransformer<ResponseT, ResultT> {
     /**
      * Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error,
      * the SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an
-     * exception will be thrown.
+     * exception will be thrown. All nonexistent parent directories will be created.
      *
      * @param path        Path to file to write to.
      * @param <ResponseT> Pojo Response type.
@@ -124,7 +124,7 @@ static <ResponseT> AsyncResponseTransformer<ResponseT, ResponseT> toFile(Path pa
     /**
      * Creates an {@link AsyncResponseTransformer} that writes all the content to the given file. In the event of an error,
      * the SDK will attempt to delete the file (whatever has been written to it so far). If the file already exists, an
-     * exception will be thrown.
+     * exception will be thrown. All nonexistent parent directories will be created.
      *
      * @param file        File to write to.
      * @param <ResponseT> Pojo Response type.

core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformer.java

@@ -32,6 +32,7 @@
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.async.SdkPublisher;
+import software.amazon.awssdk.core.exception.SdkClientException;
 
 /**
  * {@link AsyncResponseTransformer} that writes the data to the specified file.
@@ -71,9 +72,26 @@ public void onResponse(ResponseT response) {
 
     @Override
     public void onStream(SdkPublisher<ByteBuffer> publisher) {
-        // onStream may be called multiple times so reset the file channel every time
-        this.fileChannel = invokeSafely(() -> createChannel(path));
-        publisher.subscribe(new FileSubscriber(this.fileChannel, path, cf, this::exceptionOccurred));
+        try {
+            createParentDirectoriesIfNeeded(path);
+            // onStream may be called multiple times so reset the file channel every time
+            this.fileChannel = invokeSafely(() -> createChannel(path));
+            publisher.subscribe(new FileSubscriber(this.fileChannel, path, cf, this::exceptionOccurred));
+        } catch (Throwable exception) {
+            exceptionOccurred(exception);
+        }
+
+    }
+
+    private static void createParentDirectoriesIfNeeded(Path destinationPath) {
+        Path parentDirectory = destinationPath.getParent();
+        try {
+            if (parentDirectory != null) {
+                Files.createDirectories(parentDirectory);
+            }
+        } catch (IOException e) {
+            throw SdkClientException.create("Failed to create parent directories for " + destinationPath, e);
+        }
     }
 
     @Override

core/sdk-core/src/main/java/software/amazon/awssdk/core/sync/ResponseTransformer.java

@@ -93,7 +93,7 @@ default boolean needsConnectionLeftOpen() {
 
     /**
      * Creates a response transformer that writes all response content to the specified file. If the file already exists
-     * then a {@link java.nio.file.FileAlreadyExistsException} will be thrown.
+     * then a {@link java.nio.file.FileAlreadyExistsException} will be thrown. All nonexistent parent directories will be created.
      *
      * @param path        Path to file to write to.
      * @param <ResponseT> Type of unmarshalled response POJO.
@@ -103,6 +103,11 @@ static <ResponseT> ResponseTransformer<ResponseT, ResponseT> toFile(Path path) {
         return (resp, in) -> {
             try {
                 InterruptMonitor.checkInterrupted();
+                Path parentDirectory = path.getParent();
+                if (parentDirectory != null) {
+                    Files.createDirectories(parentDirectory);
+                }
+
                 Files.copy(in, path);
                 return resp;
             } catch (IOException copyException) {
@@ -135,7 +140,7 @@ static <ResponseT> ResponseTransformer<ResponseT, ResponseT> toFile(Path path) {
 
     /**
      * Creates a response transformer that writes all response content to the specified file. If the file already exists
-     * then a {@link java.nio.file.FileAlreadyExistsException} will be thrown.
+     * then a {@link java.nio.file.FileAlreadyExistsException} will be thrown. All nonexistent parent directories will be created.
      *
      * @param file        File to write to.
      * @param <ResponseT> Type of unmarshalled response POJO.

core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java

@@ -76,6 +76,19 @@ public void cancel() {
         assertThat(prepareFuture.isCompletedExceptionally()).isTrue();
     }
 
+    @Test
+    public void parentDirectoryNotExists_shouldCreate() {
+        Path testPath = testFs.getPath("test/test_file.txt");
+        FileAsyncResponseTransformer xformer = new FileAsyncResponseTransformer(testPath);
+
+        CompletableFuture prepareFuture = xformer.prepare();
+
+        xformer.onResponse(new Object());
+        xformer.onStream(new TestPublisher());
+
+        assertThat(prepareFuture.isCompletedExceptionally()).isFalse();
+    }
+
     @Test
     public void synchronousPublisher_shouldNotHang() throws Exception {
         List<CompletableFuture> futures = new ArrayList<>();

services/s3/pom.xml

@@ -136,5 +136,10 @@
             <artifactId>eventstream</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.google.jimfs</groupId>
+            <artifactId>jimfs</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

services/s3/src/it/java/software/amazon/awssdk/services/s3/GetObjectAsyncIntegrationTest.java

@@ -19,12 +19,15 @@
 import static org.junit.Assert.assertEquals;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
 
+import com.google.common.jimfs.Jimfs;
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -45,7 +48,7 @@
 import software.amazon.awssdk.utils.ImmutableMap;
 
 public class GetObjectAsyncIntegrationTest extends S3IntegrationTestBase {
-
+    private static FileSystem testFs;
     private static final String BUCKET = temporaryBucketName(GetObjectAsyncIntegrationTest.class);
 
     private static final String KEY = "some-key";
@@ -60,6 +63,7 @@ public class GetObjectAsyncIntegrationTest extends S3IntegrationTestBase {
     @BeforeClass
     public static void setupFixture() throws IOException {
         createBucket(BUCKET);
+        testFs = Jimfs.newFileSystem();
         file = new RandomTempFile(10_000);
         s3Async.putObject(PutObjectRequest.builder()
                                           .bucket(BUCKET)
@@ -70,9 +74,10 @@ public static void setupFixture() throws IOException {
     }
 
     @AfterClass
-    public static void tearDownFixture() {
+    public static void tearDownFixture() throws IOException {
         deleteBucketAndAllContents(BUCKET);
         file.delete();
+        testFs.close();
     }
 
     @Test
@@ -87,6 +92,13 @@ public void toFile() throws Exception {
         }
     }
 
+    @Test
+    public void toFile_parentDirectoriesDoNotExist_shouldSucceed() throws Exception {
+        Path path = testFs.getPath(UUID.randomUUID().toString(), "test1", "test2", "test.txt");
+        s3Async.getObject(getObjectRequest, path).join();
+        assertThat(path).hasSameBinaryContentAs(file.toPath());
+    }
+
     @Test
     public void dumpToString() throws IOException {
         String returned = s3Async.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join().asUtf8String();

services/s3/src/it/java/software/amazon/awssdk/services/s3/GetObjectIntegrationTest.java

@@ -19,11 +19,14 @@
 import static org.junit.Assert.assertEquals;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
 
+import com.google.common.jimfs.Jimfs;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.util.Properties;
+import java.util.UUID;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -39,7 +42,7 @@
 import software.amazon.awssdk.testutils.RandomTempFile;
 
 public class GetObjectIntegrationTest extends S3IntegrationTestBase {
-
+    private static FileSystem testFs;
     private static final String BUCKET = temporaryBucketName(GetObjectIntegrationTest.class);
 
     private static final String KEY = "some-key";
@@ -55,6 +58,7 @@ public class GetObjectIntegrationTest extends S3IntegrationTestBase {
     @BeforeClass
     public static void setupFixture() throws IOException {
         createBucket(BUCKET);
+        testFs = Jimfs.newFileSystem();
         file = new RandomTempFile(10_000);
         s3.putObject(PutObjectRequest.builder()
                                      .bucket(BUCKET)
@@ -63,9 +67,10 @@ public static void setupFixture() throws IOException {
     }
 
     @AfterClass
-    public static void tearDownFixture() {
+    public static void tearDownFixture() throws IOException {
         deleteBucketAndAllContents(BUCKET);
         file.delete();
+        testFs.close();
     }
 
     @Test
@@ -98,6 +103,13 @@ public void toFile() throws Exception {
         }
     }
 
+    @Test
+    public void toFile_parentDirectoriesDoNotExist_shouldSucceed() throws Exception {
+        Path path = testFs.getPath(UUID.randomUUID().toString(), "test1", "test2", "test.txt");
+        s3.getObject(getObjectRequest, path);
+        assertThat(path).hasSameBinaryContentAs(file.toPath());
+    }
+
     @Test
     public void toOutputStream() throws Exception {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();