Skip to content

Transfer Manager tests refactoring #3420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 9, 2022
Merged

Transfer Manager tests refactoring #3420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7672f75
Remove use of Junit4, clean up and consolidate tests in tm module
zoewangg Sep 9, 2022
5a5e540
Ignoring the test if unicode can't be used as directory name
zoewangg Sep 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions services-custom/s3-transfer-manager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,21 +111,42 @@
<artifactId>service-test-utils</artifactId>
<version>${awsjavasdk.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>test-utils</artifactId>
<version>${awsjavasdk.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit5.version}</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
14 changes: 14 additions & 0 deletions ...ransfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
Expand Down Expand Up @@ -52,6 +53,10 @@ public class S3IntegrationTestBase extends AwsTestBase {

protected static S3AsyncClient s3Async;

protected static S3AsyncClient s3CrtAsync;

protected static S3TransferManager tm;

/**
* Loads the AWS account info for the integration tests and creates an S3
* client for tests to use.
Expand All @@ -62,12 +67,21 @@ public static void setUp() throws Exception {
System.setProperty("aws.crt.debugnative", "true");
s3 = s3ClientBuilder().build();
s3Async = s3AsyncClientBuilder().build();
s3CrtAsync = S3CrtAsyncClient.builder()
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.build();
tm = S3TransferManager.builder()
.s3AsyncClient(s3CrtAsync)
.build();
}

@AfterAll
public static void cleanUp() {
s3.close();
s3Async.close();
s3CrtAsync.close();
tm.close();
CrtResource.waitForNoResources();
}

Expand Down
14 changes: 0 additions & 14 deletions .../src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerCopyIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.transfer.s3.model.CompletedCopy;
import software.amazon.awssdk.transfer.s3.model.Copy;
Expand All @@ -42,26 +40,14 @@ public class S3TransferManagerCopyIntegrationTest extends S3IntegrationTestBase
private static final String COPIED_OBJ_SPECIAL_CHARACTER = "special-special-chars-@$%";
private static final long OBJ_SIZE = ThreadLocalRandom.current().nextLong(8 * MB, 16 * MB + 1);

private static S3TransferManager tm;

private static S3AsyncClient s3AsyncClient;

@BeforeAll
public static void setUp() throws Exception {
S3IntegrationTestBase.setUp();
createBucket(BUCKET);
s3AsyncClient = S3CrtAsyncClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.maxConcurrency(100)
.build();
tm = S3TransferManager.builder()
.s3AsyncClient(s3AsyncClient)
.build();
}

@AfterAll
public static void teardown() throws Exception {
s3AsyncClient.close();
tm.close();
deleteBucketAndAllContents(BUCKET);
S3IntegrationTestBase.cleanUp();
Expand Down
14 changes: 2 additions & 12 deletions ...software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,14 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.ComparisonFailure;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient;
import org.opentest4j.AssertionFailedError;
import software.amazon.awssdk.testutils.FileUtils;
import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload;
import software.amazon.awssdk.transfer.s3.model.DirectoryDownload;
Expand All @@ -50,7 +49,6 @@ public class S3TransferManagerDownloadDirectoryIntegrationTest extends S3Integra
+ "-delimiter");
private static final String CUSTOM_DELIMITER = "-";

private static S3TransferManager tm;
private static Path sourceDirectory;
private Path directory;

Expand All @@ -61,14 +59,6 @@ public static void setUp() throws Exception {
createBucket(TEST_BUCKET_CUSTOM_DELIMITER);
sourceDirectory = createLocalTestDirectory();

tm = S3TransferManager.builder()
.s3AsyncClient(S3CrtAsyncClient.builder()
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.maxConcurrency(100)
.build())
.build();

tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory).bucket(TEST_BUCKET)).completionFuture().join();

tm.uploadDirectory(u -> u.sourceDirectory(sourceDirectory)
Expand Down Expand Up @@ -236,7 +226,7 @@ private static void assertLeftHasRight(Path left, Path right) {
try {
assertThat(rightPath).exists();
} catch (AssertionError e) {
throw new ComparisonFailure(e.getMessage(), toFileTreeString(left), toFileTreeString(right));
throw new AssertionFailedError(e.getMessage(), toFileTreeString(left), toFileTreeString(right));
}
if (Files.isRegularFile(leftPath)) {
assertThat(leftPath).hasSameBinaryContentAs(rightPath);
Expand Down
7 changes: 0 additions & 7 deletions .../it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ public class S3TransferManagerDownloadIntegrationTest extends S3IntegrationTestB
private static final String BUCKET = temporaryBucketName(S3TransferManagerDownloadIntegrationTest.class);
private static final String KEY = "key";
private static final int OBJ_SIZE = 16 * 1024 * 1024;
private static S3TransferManager tm;
private static File file;

@BeforeAll
Expand All @@ -60,17 +59,11 @@ public static void setup() throws IOException {
.bucket(BUCKET)
.key(KEY)
.build(), file.toPath());
tm = S3TransferManager.builder()
.s3AsyncClient(S3CrtAsyncClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.build())
.build();
}

@AfterAll
public static void cleanup() {
deleteBucketAndAllContents(BUCKET);
tm.close();
S3IntegrationTestBase.cleanUp();
}

Expand Down
8 changes: 0 additions & 8 deletions ...ftware/amazon/awssdk/transfer/s3/S3TransferManagerDownloadPauseResumeIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ public class S3TransferManagerDownloadPauseResumeIntegrationTest extends S3Integ
private static final String KEY = "key";
// 24 * MB is chosen to make sure we have data written in the file already upon pausing.
private static final long OBJ_SIZE = 24 * MB;
private static S3TransferManager tm;
private static File sourceFile;

@BeforeAll
Expand All @@ -63,18 +62,11 @@ public static void setup() throws Exception {
.bucket(BUCKET)
.key(KEY)
.build(), sourceFile.toPath());
tm =
S3TransferManager.builder().s3AsyncClient(S3AsyncClient.builder()
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
.region(DEFAULT_REGION)
.build())
.build();
}

@AfterAll
public static void cleanup() {
deleteBucketAndAllContents(BUCKET);
tm.close();
sourceFile.delete();
S3IntegrationTestBase.cleanUp();
}
Expand Down
Loading