Skip to content

[draft] java-based integ test #5129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 1, 2024
Merged

[draft] java-based integ test #5129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d37ef50
Add java-based transfer manager to integration test suite
L-Applin Apr 22, 2024
85553fa
Merge branch 'feature/master/s3mpu' into olapplin/tm-java-integ-tests
L-Applin Apr 22, 2024
7accd09
Add stability tests
L-Applin Apr 23, 2024
2f953bb
Merge remote-tracking branch 'origin/olapplin/tm-java-integ-tests' in…
L-Applin Apr 23, 2024
d590726
Merge branch 'feature/master/s3mpu' into olapplin/tm-java-integ-tests
L-Applin Apr 26, 2024
7ee7506
Merge branch 'feature/master/s3mpu' into olapplin/tm-java-integ-tests
L-Applin May 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions ...ransfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@

package software.amazon.awssdk.transfer.s3;

import java.util.stream.Stream;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.provider.Arguments;
import software.amazon.awssdk.crt.CrtResource;
import software.amazon.awssdk.crt.Log;
import software.amazon.awssdk.regions.Region;
Expand Down Expand Up @@ -68,19 +70,17 @@ public static void setUpForAllIntegTests() throws Exception {
Log.initLoggingToStdout(Log.LogLevel.Warn);
System.setProperty("aws.crt.debugnative", "true");
s3 = s3ClientBuilder().build();
s3Async = s3AsyncClientBuilder()
.multipartEnabled(true)
.build();
s3Async = s3AsyncClientBuilder().build();
s3CrtAsync = S3CrtAsyncClient.builder()
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.build();
tmCrt = S3TransferManager.builder()
.s3Client(s3CrtAsync)
.build();
tmJava = S3TransferManager.builder()
.s3Client(s3Async)
.s3Client(s3CrtAsync)
.build();
tmJava = S3TransferManager.builder()
.s3Client(s3Async)
.build();

}

Expand All @@ -101,6 +101,7 @@ protected static S3ClientBuilder s3ClientBuilder() {

protected static S3AsyncClientBuilder s3AsyncClientBuilder() {
return S3AsyncClient.builder()
.multipartEnabled(true)
.region(DEFAULT_REGION)
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN);
}
Expand Down Expand Up @@ -174,4 +175,10 @@ protected static void deleteBucketAndAllContents(String bucketName) {
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
}

static Stream<Arguments> transferManagers() {
return Stream.of(
Arguments.of(tmCrt),
Arguments.of(tmJava));
}

}
7 changes: 0 additions & 7 deletions .../src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerCopyIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,6 @@ enum TmType{
JAVA, CRT
}

private static Stream<Arguments> transferManagers() {
return Stream.of(
Arguments.of(TmType.JAVA),
Arguments.of(TmType.CRT)
);
}

@ParameterizedTest
@MethodSource("transferManagers")
void copy_copiedObject_hasSameContent(TmType tmType) throws Exception {
Expand Down
65 changes: 41 additions & 24 deletions ...software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.opentest4j.AssertionFailedError;
import software.amazon.awssdk.testutils.FileUtils;
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
Expand Down Expand Up @@ -97,6 +97,7 @@ public static void teardown() {
}

closeQuietly(tmCrt, log.logger());
closeQuietly(tmJava, log.logger());
}

/**
Expand All @@ -116,21 +117,23 @@ public static void teardown() {
* }
* </pre>
*/
@Test
public void downloadDirectory() throws Exception {
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
.bucket(TEST_BUCKET));
@ParameterizedTest
@MethodSource("transferManagers")
public void downloadDirectory(S3TransferManager tm) throws Exception {
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destination(directory)
.bucket(TEST_BUCKET));
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
assertTwoDirectoriesHaveSameStructure(sourceDirectory, directory);
}

@ParameterizedTest
@ValueSource(strings = {"notes/2021", "notes/2021/"})
void downloadDirectory_withPrefix(String prefix) throws Exception {
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
.bucket(TEST_BUCKET));
@MethodSource("prefixTestArguments")
void downloadDirectory_withPrefix(S3TransferManager tm, String prefix) throws Exception {
DirectoryDownload downloadDirectory =
tm.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
.bucket(TEST_BUCKET));
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();

Expand All @@ -152,12 +155,14 @@ void downloadDirectory_withPrefix(String prefix) throws Exception {
* }
* </pre>
*/
@Test
void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly() throws Exception {
@ParameterizedTest
@MethodSource("transferManagers")
void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly(S3TransferManager tm)
throws Exception {
String prefix = "notes";
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
.bucket(TEST_BUCKET));
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.prefix(prefix))
.bucket(TEST_BUCKET));
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();

Expand All @@ -182,14 +187,15 @@ void downloadDirectory_containsObjectWithPrefixInTheKey_shouldResolveCorrectly()
* }
* </pre>
*/
@Test
public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
@ParameterizedTest
@MethodSource("transferManagers")
public void downloadDirectory_withPrefixAndDelimiter(S3TransferManager tm) throws Exception {
String prefix = "notes-2021";
DirectoryDownload downloadDirectory =
tmCrt.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.delimiter(CUSTOM_DELIMITER)
tm.downloadDirectory(u -> u.destination(directory)
.listObjectsV2RequestTransformer(r -> r.delimiter(CUSTOM_DELIMITER)
.prefix(prefix))
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
.bucket(TEST_BUCKET_CUSTOM_DELIMITER));
CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve("notes").resolve("2021"), directory);
Expand All @@ -206,9 +212,10 @@ public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
* }
* </pre>
*/
@Test
public void downloadDirectory_withFilter() throws Exception {
DirectoryDownload downloadDirectory = tmCrt.downloadDirectory(u -> u
@ParameterizedTest
@MethodSource("transferManagers")
public void downloadDirectory_withFilter(S3TransferManager tm) throws Exception {
DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u
.destination(directory)
.bucket(TEST_BUCKET)
.filter(s3Object -> s3Object.key().startsWith("notes/2021/2")));
Expand Down Expand Up @@ -296,4 +303,14 @@ private static Path createLocalTestDirectory() throws IOException {
RandomStringUtils.random(100).getBytes(StandardCharsets.UTF_8));
return directory;
}

private static Stream<Arguments> prefixTestArguments() {
String[] prefixes = {"notes/2021", "notes/2021/"};
return Stream.of(
Arguments.of(tmCrt, prefixes[0]),
Arguments.of(tmCrt, prefixes[1]),
Arguments.of(tmJava, prefixes[0]),
Arguments.of(tmJava, prefixes[1])
);
}
}
55 changes: 30 additions & 25 deletions .../it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.core.async.ResponsePublisher;
Expand Down Expand Up @@ -66,11 +67,12 @@ public static void cleanup() {
deleteBucketAndAllContents(BUCKET);
}

@Test
void download_toFile() throws Exception {
@ParameterizedTest
@MethodSource("transferManagers")
void download_toFile(S3TransferManager tm) throws Exception {
Path path = RandomTempFile.randomUncreatedFile().toPath();
FileDownload download =
tmCrt.downloadFile(DownloadFileRequest.builder()
tm.downloadFile(DownloadFileRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.destination(path)
.addTransferListener(LoggingTransferListener.create())
Expand All @@ -80,44 +82,47 @@ void download_toFile() throws Exception {
assertThat(completedFileDownload.response().responseMetadata().requestId()).isNotNull();
}

@Test
void download_toFile_shouldReplaceExisting() throws IOException {
@ParameterizedTest
@MethodSource("transferManagers")
void download_toFile_shouldReplaceExisting(S3TransferManager tm) throws IOException {
Path path = RandomTempFile.randomUncreatedFile().toPath();
Files.write(path, RandomStringUtils.random(1024).getBytes(StandardCharsets.UTF_8));
assertThat(path).exists();
FileDownload download =
tmCrt.downloadFile(DownloadFileRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.destination(path)
.addTransferListener(LoggingTransferListener.create())
.build());
tm.downloadFile(DownloadFileRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.destination(path)
.addTransferListener(LoggingTransferListener.create())
.build());
CompletedFileDownload completedFileDownload = download.completionFuture().join();
assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file));
assertThat(completedFileDownload.response().responseMetadata().requestId()).isNotNull();
}

@Test
void download_toBytes() throws Exception {
@ParameterizedTest
@MethodSource("transferManagers")
void download_toBytes(S3TransferManager tm) throws Exception {
Download<ResponseBytes<GetObjectResponse>> download =
tmCrt.download(DownloadRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.responseTransformer(AsyncResponseTransformer.toBytes())
.addTransferListener(LoggingTransferListener.create())
.build());
tm.download(DownloadRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.responseTransformer(AsyncResponseTransformer.toBytes())
.addTransferListener(LoggingTransferListener.create())
.build());
CompletedDownload<ResponseBytes<GetObjectResponse>> completedDownload = download.completionFuture().join();
ResponseBytes<GetObjectResponse> result = completedDownload.result();
assertThat(Md5Utils.md5AsBase64(result.asByteArray())).isEqualTo(Md5Utils.md5AsBase64(file));
assertThat(result.response().responseMetadata().requestId()).isNotNull();
}

@Test
void download_toPublisher() throws Exception {
@ParameterizedTest
@MethodSource("transferManagers")
void download_toPublisher(S3TransferManager tm) throws Exception {
Download<ResponsePublisher<GetObjectResponse>> download =
tmCrt.download(DownloadRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.responseTransformer(AsyncResponseTransformer.toPublisher())
.addTransferListener(LoggingTransferListener.create())
.build());
tm.download(DownloadRequest.builder()
.getObjectRequest(b -> b.bucket(BUCKET).key(KEY))
.responseTransformer(AsyncResponseTransformer.toPublisher())
.addTransferListener(LoggingTransferListener.create())
.build());
CompletedDownload<ResponsePublisher<GetObjectResponse>> completedDownload = download.completionFuture().join();
ResponsePublisher<GetObjectResponse> responsePublisher = completedDownload.result();
ByteBuffer buf = ByteBuffer.allocate(Math.toIntExact(responsePublisher.response().contentLength()));
Expand Down
Loading