Skip to content

Commit 103d2d4

Browse files
zoewanggzoewangg
authored
[TM DownloadDirectory Part2] Implement download directory in transfer manager (#3010)
* Implement download directory in transfer manager * Add more tests and address comments * Remove create parent directories logic and add changelog entry
1 parent 6aaec97 commit 103d2d4

23 files changed

+1185
-282
lines changed

‎.changes/next-release/feature-S3TransferManager-4aba25e.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"category": "S3 Transfer Manager",
3+
"contributor": "",
4+
"type": "feature",
5+
"description": "Implement downloadDirectory API in the S3TransferManager"
6+
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.transfer.s3;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
20+
import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
21+
22+
import java.io.IOException;
23+
import java.io.UncheckedIOException;
24+
import java.nio.charset.StandardCharsets;
25+
import java.nio.file.FileVisitResult;
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.nio.file.Paths;
29+
import java.nio.file.SimpleFileVisitor;
30+
import java.nio.file.attribute.BasicFileAttributes;
31+
import org.apache.commons.lang3.RandomStringUtils;
32+
import org.junit.After;
33+
import org.junit.AfterClass;
34+
import org.junit.Before;
35+
import org.junit.BeforeClass;
36+
import org.junit.Test;
37+
import software.amazon.awssdk.testutils.FileUtils;
38+
import software.amazon.awssdk.utils.Logger;
39+
40+
public class S3TransferManagerDownloadDirectoryIntegrationTest extends S3IntegrationTestBase {
41+
private static final Logger log = Logger.loggerFor(S3TransferManagerDownloadDirectoryIntegrationTest.class);
42+
private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class);
43+
private static final String TEST_BUCKET_CUSTOM_DELIMITER = temporaryBucketName("S3TransferManagerUploadIntegrationTest"
44+
+ "-delimiter");
45+
private static final String CUSTOM_DELIMITER = "-";
46+
47+
private static S3TransferManager tm;
48+
private static Path sourceDirectory;
49+
private Path destinationDirectory;
50+
51+
@BeforeClass
52+
public static void setUp() throws Exception {
53+
S3IntegrationTestBase.setUp();
54+
createBucket(TEST_BUCKET);
55+
createBucket(TEST_BUCKET_CUSTOM_DELIMITER);
56+
sourceDirectory = createLocalTestDirectory();
57+
58+
tm = S3TransferManager.builder()
59+
.s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
60+
.region(DEFAULT_REGION)
61+
.maxConcurrency(100))
62+
.build();
63+
64+
tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory).bucket(TEST_BUCKET)).completionFuture().join();
65+
66+
tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory)
67+
.delimiter(CUSTOM_DELIMITER)
68+
.bucket(TEST_BUCKET_CUSTOM_DELIMITER))
69+
.completionFuture().join();
70+
}
71+
72+
@Before
73+
public void setUpPerTest() throws IOException {
74+
destinationDirectory = Files.createTempDirectory("destination");
75+
}
76+
77+
@After
78+
public void cleanup() {
79+
FileUtils.cleanUpTestDirectory(destinationDirectory);
80+
}
81+
82+
@AfterClass
83+
public static void teardown() {
84+
try {
85+
FileUtils.cleanUpTestDirectory(sourceDirectory);
86+
} catch (Exception exception) {
87+
log.warn(() -> "Failed to clean up test directory " + sourceDirectory, exception);
88+
}
89+
90+
try {
91+
deleteBucketAndAllContents(TEST_BUCKET);
92+
} catch (Exception exception) {
93+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET, exception);
94+
}
95+
96+
try {
97+
deleteBucketAndAllContents(TEST_BUCKET_CUSTOM_DELIMITER);
98+
} catch (Exception exception) {
99+
log.warn(() -> "Failed to delete s3 bucket " + TEST_BUCKET_CUSTOM_DELIMITER, exception);
100+
}
101+
102+
closeQuietly(tm, log.logger());
103+
S3IntegrationTestBase.cleanUp();
104+
}
105+
106+
@Test
107+
public void downloadDirectory() {
108+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
109+
.bucket(TEST_BUCKET));
110+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().join();
111+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
112+
assertTwoDirectoriesHaveSameStructure(sourceDirectory, destinationDirectory);
113+
}
114+
115+
@Test
116+
public void downloadDirectory_withPrefix() {
117+
String prefix = "notes";
118+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
119+
.prefix(prefix)
120+
.bucket(TEST_BUCKET));
121+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().join();
122+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
123+
124+
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve(prefix), destinationDirectory.resolve(prefix));
125+
}
126+
127+
@Test
128+
public void downloadDirectory_withDelimiter() {
129+
String prefix = "notes";
130+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destinationDirectory(destinationDirectory)
131+
.delimiter(CUSTOM_DELIMITER)
132+
.prefix(prefix)
133+
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
134+
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().join();
135+
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
136+
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve(prefix), destinationDirectory.resolve(prefix));
137+
}
138+
139+
private static void assertTwoDirectoriesHaveSameStructure(Path path, Path otherPath) {
140+
try {
141+
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
142+
@Override
143+
public FileVisitResult visitFile(Path file,
144+
BasicFileAttributes attrs)
145+
throws IOException {
146+
FileVisitResult result = super.visitFile(file, attrs);
147+
148+
Path relativePath = path.relativize(file);
149+
Path otherFile = otherPath.resolve(relativePath);
150+
log.debug(() -> String.format("Comparing %s with %s", file, otherFile));
151+
assertThat(file).hasSameBinaryContentAs(otherFile);
152+
return result;
153+
}
154+
});
155+
} catch (IOException e) {
156+
throw new UncheckedIOException(String.format("Failed to compare %s with %s", path, otherPath), e);
157+
}
158+
}
159+
160+
/**
161+
* Create a test directory with the following structure
162+
* <pre>
163+
* {@code
164+
* - source
165+
* - README.md
166+
* - CHANGELOG.md
167+
* - notes
168+
* - 2021
169+
* - 1.txt
170+
* - 2.txt
171+
* - 2022
172+
* - 1.txt
173+
* - important.txt
174+
* }
175+
* </pre>
176+
*/
177+
private static Path createLocalTestDirectory() throws IOException {
178+
Path directory = Files.createTempDirectory("source");
179+
180+
String directoryName = directory.toString();
181+
182+
Files.createDirectory(Paths.get(directoryName, "notes"));
183+
Files.createDirectory(Paths.get(directoryName, "notes", "2021"));
184+
Files.createDirectory(Paths.get(directoryName, "notes", "2022"));
185+
Files.write(Paths.get(directoryName, "README.md"), RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
186+
Files.write(Paths.get(directoryName, "CHANGELOG.md"), RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
187+
Files.write(Paths.get(directoryName, "notes", "2021", "1.txt"),
188+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
189+
Files.write(Paths.get(directoryName, "notes", "2021", "2.txt"),
190+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
191+
Files.write(Paths.get(directoryName, "notes", "2022", "1.txt"),
192+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
193+
Files.write(Paths.get(directoryName, "notes", "important.txt"),
194+
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
195+
return directory;
196+
}
197+
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadDirectoryOverrideConfiguration.java

Lines changed: 0 additions & 178 deletions
This file was deleted.

0 commit comments

Comments
 (0)