Skip to content

Commit 97f0a5f

Browse files
L-ApplinL-Applin
authored
[draft] java-based integ test (#5129)
* Add java-based transfer manager to integration test suite * Add stability tests
1 parent d5dff64 commit 97f0a5f

9 files changed

+255
-173
lines changed

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515

1616
package software.amazon.awssdk.transfer.s3;
1717

18+
import java.util.stream.Stream;
1819
import org.junit.jupiter.api.AfterAll;
1920
import org.junit.jupiter.api.BeforeAll;
21+
import org.junit.jupiter.params.provider.Arguments;
2022
import software.amazon.awssdk.crt.CrtResource;
2123
import software.amazon.awssdk.crt.Log;
2224
import software.amazon.awssdk.regions.Region;
@@ -68,19 +70,17 @@ public static void setUpForAllIntegTests() throws Exception {
6870
Log.initLoggingToStdout(Log.LogLevel.Warn);
6971
System.setProperty("aws.crt.debugnative", "true");
7072
s3 = s3ClientBuilder().build();
71-
s3Async = s3AsyncClientBuilder()
72-
.multipartEnabled(true)
73-
.build();
73+
s3Async = s3AsyncClientBuilder().build();
7474
s3CrtAsync = S3CrtAsyncClient.builder()
7575
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
7676
.region(DEFAULT_REGION)
7777
.build();
7878
tmCrt = S3TransferManager.builder()
79-
.s3Client(s3CrtAsync)
80-
.build();
81-
tmJava = S3TransferManager.builder()
82-
.s3Client(s3Async)
79+
.s3Client(s3CrtAsync)
8380
.build();
81+
tmJava = S3TransferManager.builder()
82+
.s3Client(s3Async)
83+
.build();
8484

8585
}
8686

@@ -101,6 +101,7 @@ protected static S3ClientBuilder s3ClientBuilder() {
101101

102102
protected static S3AsyncClientBuilder s3AsyncClientBuilder() {
103103
return S3AsyncClient.builder()
104+
.multipartEnabled(true)
104105
.region(DEFAULT_REGION)
105106
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
106107
}
@@ -174,4 +175,10 @@ protected static void deleteBucketAndAllContents(String bucketName) {
174175
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
175176
}
176177

178+
static Stream<Arguments> transferManagers() {
179+
return Stream.of(
180+
Arguments.of(tmCrt),
181+
Arguments.of(tmJava));
182+
}
183+
177184
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerCopyIntegrationTest.java

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,6 @@ enum TmType{
5757
JAVA, CRT
5858
}
5959

60-
private static Stream<Arguments> transferManagers() {
61-
return Stream.of(
62-
Arguments.of(TmType.JAVA),
63-
Arguments.of(TmType.CRT)
64-
);
65-
}
66-
6760
@ParameterizedTest
6861
@MethodSource("transferManagers")
6962
void copy_copiedObject_hasSameContent(TmType tmType) throws Exception {

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java

Lines changed: 41 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@
3333
import org.junit.jupiter.api.AfterEach;
3434
import org.junit.jupiter.api.BeforeAll;
3535
import org.junit.jupiter.api.BeforeEach;
36-
import org.junit.jupiter.api.Test;
3736
import org.junit.jupiter.params.ParameterizedTest;
38-
import org.junit.jupiter.params.provider.ValueSource;
37+
import org.junit.jupiter.params.provider.Arguments;
38+
import org.junit.jupiter.params.provider.MethodSource;
3939
import org.opentest4j.AssertionFailedError;
4040
import software.amazon.awssdk.testutils.FileUtils;
4141
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
@@ -97,6 +97,7 @@ public static void teardown() {
9797
}
9898

9999
closeQuietly(tmCrt, log.logger());
100+
closeQuietly(tmJava, log.logger());
100101
}
101102

102103
/**
@@ -116,21 +117,23 @@ public static void teardown() {
116117
* }
117118
* </pre>
118119
*/
119-
@Test
120-
public void downloadDirectory() throws Exception {
121-
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
122-
.bucket(TEST_BUCKET));
120+
@ParameterizedTest
121+
@MethodSource("transferManagers")
122+
public void downloadDirectory(S3TransferManager tm) throws Exception {
123+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destination(directory)
124+
.bucket(TEST_BUCKET));
123125
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
124126
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
125127
assertTwoDirectoriesHaveSameStructure(sourceDirectory, directory);
126128
}
127129

128130
@ParameterizedTest
129-
@ValueSource(strings = {"notes/2021", "notes/2021/"})
130-
void downloadDirectory_withPrefix(String prefix) throws Exception {
131-
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
132-
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
133-
.bucket(TEST_BUCKET));
131+
@MethodSource("prefixTestArguments")
132+
void downloadDirectory_withPrefix(S3TransferManager tm, String prefix) throws Exception {
133+
DirectoryDownload downloadDirectory =
134+
tm.downloadDirectory(u -> u.destination(directory)
135+
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
136+
.bucket(TEST_BUCKET));
134137
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
135138
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
136139

@@ -152,12 +155,14 @@ void downloadDirectory_withPrefix(String prefix) throws Exception {
152155
* }
153156
* </pre>
154157
*/
155-
@Test
156-
void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly() throws Exception {
158+
@ParameterizedTest
159+
@MethodSource("transferManagers")
160+
void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly(S3TransferManager tm)
161+
throws Exception {
157162
String prefix = "notes";
158-
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
159-
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
160-
.bucket(TEST_BUCKET));
163+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destination(directory)
164+
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
165+
.bucket(TEST_BUCKET));
161166
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
162167
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
163168

@@ -182,14 +187,15 @@ void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly()
182187
* }
183188
* </pre>
184189
*/
185-
@Test
186-
public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
190+
@ParameterizedTest
191+
@MethodSource("transferManagers")
192+
public void downloadDirectory_withPrefixAndDelimiter(S3TransferManager tm) throws Exception {
187193
String prefix = "notes-2021";
188194
DirectoryDownload downloadDirectory =
189-
tmCrt.downloadDirectory(u -> u.destination(directory)
190-
.listObjectsV2RequestTransformer(r -> r.delimiter(CUSTOM_DELIMITER)
195+
tm.downloadDirectory(u -> u.destination(directory)
196+
.listObjectsV2RequestTransformer(r -> r.delimiter(CUSTOM_DELIMITER)
191197
.prefix(prefix))
192-
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
198+
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
193199
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
194200
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
195201
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve("notes").resolve("2021"), directory);
@@ -206,9 +212,10 @@ public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
206212
* }
207213
* </pre>
208214
*/
209-
@Test
210-
public void downloadDirectory_withFilter() throws Exception {
211-
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u
215+
@ParameterizedTest
216+
@MethodSource("transferManagers")
217+
public void downloadDirectory_withFilter(S3TransferManager tm) throws Exception {
218+
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u
212219
.destination(directory)
213220
.bucket(TEST_BUCKET)
214221
.filter(s3Object -> s3Object.key().startsWith("notes/2021/2")));
@@ -296,4 +303,14 @@ private static Path createLocalTestDirectory() throws IOException {
296303
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
297304
return directory;
298305
}
306+
307+
private static Stream<Arguments> prefixTestArguments() {
308+
String[] prefixes = {"notes/2021", "notes/2021/"};
309+
return Stream.of(
310+
Arguments.of(tmCrt, prefixes[0]),
311+
Arguments.of(tmCrt, prefixes[1]),
312+
Arguments.of(tmJava, prefixes[0]),
313+
Arguments.of(tmJava, prefixes[1])
314+
);
315+
}
299316
}

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
import org.apache.commons.lang3.RandomStringUtils;
3030
import org.junit.jupiter.api.AfterAll;
3131
import org.junit.jupiter.api.BeforeAll;
32-
import org.junit.jupiter.api.Test;
32+
import org.junit.jupiter.params.ParameterizedTest;
33+
import org.junit.jupiter.params.provider.MethodSource;
3334
import software.amazon.awssdk.core.ResponseBytes;
3435
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
3536
import software.amazon.awssdk.core.async.ResponsePublisher;
@@ -66,11 +67,12 @@ public static void cleanup() {
6667
deleteBucketAndAllContents(BUCKET);
6768
}
6869

69-
@Test
70-
void download_toFile() throws Exception {
70+
@ParameterizedTest
71+
@MethodSource("transferManagers")
72+
void download_toFile(S3TransferManager tm) throws Exception {
7173
Path path = RandomTempFile.randomUncreatedFile().toPath();
7274
FileDownload download =
73-
tmCrt.downloadFile(DownloadFileRequest.builder()
75+
tm.downloadFile(DownloadFileRequest.builder()
7476
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
7577
.destination(path)
7678
.addTransferListener(LoggingTransferListener.create())
@@ -80,44 +82,47 @@ void download_toFile() throws Exception {
8082
assertThat(completedFileDownload.response().responseMetadata().requestId()).isNotNull();
8183
}
8284

83-
@Test
84-
void download_toFile_shouldReplaceExisting() throws IOException {
85+
@ParameterizedTest
86+
@MethodSource("transferManagers")
87+
void download_toFile_shouldReplaceExisting(S3TransferManager tm) throws IOException {
8588
Path path = RandomTempFile.randomUncreatedFile().toPath();
8689
Files.write(path, RandomStringUtils.random(1024).getBytes(StandardCharsets.UTF_8));
8790
assertThat(path).exists();
8891
FileDownload download =
89-
tmCrt.downloadFile(DownloadFileRequest.builder()
90-
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
91-
.destination(path)
92-
.addTransferListener(LoggingTransferListener.create())
93-
.build());
92+
tm.downloadFile(DownloadFileRequest.builder()
93+
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
94+
.destination(path)
95+
.addTransferListener(LoggingTransferListener.create())
96+
.build());
9497
CompletedFileDownload completedFileDownload = download.completionFuture().join();
9598
assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file));
9699
assertThat(completedFileDownload.response().responseMetadata().requestId()).isNotNull();
97100
}
98101

99-
@Test
100-
void download_toBytes() throws Exception {
102+
@ParameterizedTest
103+
@MethodSource("transferManagers")
104+
void download_toBytes(S3TransferManager tm) throws Exception {
101105
Download<ResponseBytes<GetObjectResponse>> download =
102-
tmCrt.download(DownloadRequest.builder()
103-
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
104-
.responseTransformer(AsyncResponseTransformer.toBytes())
105-
.addTransferListener(LoggingTransferListener.create())
106-
.build());
106+
tm.download(DownloadRequest.builder()
107+
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
108+
.responseTransformer(AsyncResponseTransformer.toBytes())
109+
.addTransferListener(LoggingTransferListener.create())
110+
.build());
107111
CompletedDownload<ResponseBytes<GetObjectResponse>> completedDownload = download.completionFuture().join();
108112
ResponseBytes<GetObjectResponse> result = completedDownload.result();
109113
assertThat(Md5Utils.md5AsBase64(result.asByteArray())).isEqualTo(Md5Utils.md5AsBase64(file));
110114
assertThat(result.response().responseMetadata().requestId()).isNotNull();
111115
}
112116

113-
@Test
114-
void download_toPublisher() throws Exception {
117+
@ParameterizedTest
118+
@MethodSource("transferManagers")
119+
void download_toPublisher(S3TransferManager tm) throws Exception {
115120
Download<ResponsePublisher<GetObjectResponse>> download =
116-
tmCrt.download(DownloadRequest.builder()
117-
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
118-
.responseTransformer(AsyncResponseTransformer.toPublisher())
119-
.addTransferListener(LoggingTransferListener.create())
120-
.build());
121+
tm.download(DownloadRequest.builder()
122+
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
123+
.responseTransformer(AsyncResponseTransformer.toPublisher())
124+
.addTransferListener(LoggingTransferListener.create())
125+
.build());
121126
CompletedDownload<ResponsePublisher<GetObjectResponse>> completedDownload = download.completionFuture().join();
122127
ResponsePublisher<GetObjectResponse> responsePublisher = completedDownload.result();
123128
ByteBuffer buf = ByteBuffer.allocate(Math.toIntExact(responsePublisher.response().contentLength()));

0 commit comments

Comments
 (0)