added README, verify crc32 of uploaded object with an additional get object, precalculated crc32 when possible, rename S3ClientFlavor fields,

Author: L-Applin

test/s3-tests/README.md

@@ -0,0 +1,31 @@
+# SDK Regression Tests for Amazon S3
+
+## Description
+This module contains SDK regression tests for Amazon S3 streaming operations with various SDK configurations.
+
+
+## How to run
+
+### Credentials
+
+The tests require valid AWS credentials to be available in the default credential file under the `aws-test-account` profile.
+
+### Run the tests
+
+- Run from your IDE
+
+- Run from maven command line
+
+```
+mvn clean install -P s3-regression-tests -pl :stability-tests
+```
+
+## Adding New Tests
+
+- The tests are built using [JUnit 5](https://junit.org/junit5/). Make sure you are using the correct APIs and mixing of
+  Junit 4 and Junit 5 APIs on the same test can have unexpected results.
+
+- All tests should have the suffix of `RegressionTesting`, eg: ``
+
+
+

test/s3-tests/pom.xml

@@ -167,6 +167,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>test-utils</artifactId>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java renamed to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/ControlPlaneOperationRegressionTesting.java

@@ -42,8 +42,8 @@
 import software.amazon.awssdk.utils.CompletableFutureUtils;
 import software.amazon.awssdk.utils.Logger;
 
-public class DataplaneOperationRegressionTesting extends BaseS3RegressionTest {
-    private static final Logger LOG = Logger.loggerFor(DataplaneOperationRegressionTesting.class);
+public class ControlPlaneOperationRegressionTesting extends BaseS3RegressionTest {
+    private static final Logger LOG = Logger.loggerFor(ControlPlaneOperationRegressionTesting.class);
 
     // Request checksum required
     @ParameterizedTest

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java

@@ -16,7 +16,7 @@
 package software.amazon.awssdk.services.s3.regression;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static software.amazon.awssdk.services.s3.regression.DataplaneOperationRegressionTesting.testConfigs;
+import static software.amazon.awssdk.services.s3.regression.ControlPlaneOperationRegressionTesting.testConfigs;
 import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType;
 import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz;
 import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle;
@@ -134,13 +134,13 @@ void downloadObject(DownloadConfig config) throws Exception {
 
         CallResponse response;
         switch (config.baseConfig().getFlavor()) {
-            case JAVA_BASED: {
+            case STANDARD_SYNC: {
                 response = callSyncGetObject(config, request);
                 break;
             }
-            case ASYNC_JAVA_BASED:
-            case TM_JAVA:
-            case ASYNC_CRT: {
+            case STANDARD_ASYNC:
+            case MULTIPART_ENABLED:
+            case CRT_BASED: {
                 response = callAsyncGetObject(request, config);
                 break;
             }
@@ -223,18 +223,16 @@ static ObjectWithCRC uploadMultiPartObject() throws Exception {
         byte[] fullContent = os.toByteArray();
         String crc32 = crc32(fullContent);
         for (BucketType bucketType : BucketType.values()) {
-            doMultipartUpload(bucketType, name, fullContent, crc32);
+            doMultipartUpload(bucketType, name, fullContent);
         }
         return new ObjectWithCRC(name, crc32);
     }
 
-    static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content, String fullContentCRC32) {
+    static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) {
         String bucket = bucketForType(bucketType);
         LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket)
         );
         CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder()
-                                                                               // .checksumAlgorithm(ChecksumAlgorithm.CRC32)
-                                                                               // .checksumType(ChecksumType.FULL_OBJECT)
                                                                                .bucket(bucket)
                                                                                .key(objectName)
                                                                                .build();
@@ -492,7 +490,7 @@ private static Path createTempDir(String path) {
 
     private S3Client makeSyncClient(TestConfig config) {
         switch (config.getFlavor()) {
-            case JAVA_BASED:
+            case STANDARD_SYNC:
                 return S3Client.builder()
                                .forcePathStyle(config.isForcePathStyle())
                                .requestChecksumCalculation(config.getRequestChecksumValidation())
@@ -507,15 +505,15 @@ private S3Client makeSyncClient(TestConfig config) {
 
     private S3AsyncClient makeAsyncClient(TestConfig config) {
         switch (config.getFlavor()) {
-            case ASYNC_JAVA_BASED:
+            case STANDARD_ASYNC:
                 return S3AsyncClient.builder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())
                                     .region(REGION)
                                     .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
                                     .accelerate(config.isAccelerateEnabled())
                                     .build();
-            case TM_JAVA:
+            case MULTIPART_ENABLED:
                 return S3AsyncClient.builder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())
@@ -524,7 +522,7 @@ private S3AsyncClient makeAsyncClient(TestConfig config) {
                                     .accelerate(config.isAccelerateEnabled())
                                     .multipartEnabled(true)
                                     .build();
-            case ASYNC_CRT: {
+            case CRT_BASED: {
                 return S3AsyncClient.crtBuilder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java

@@ -239,7 +239,7 @@ public static String crc32(Path p) throws IOException {
 
     public static S3Client makeSyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) {
         switch (config.getFlavor()) {
-            case JAVA_BASED:
+            case STANDARD_SYNC:
                 return S3Client.builder()
                                .forcePathStyle(config.isForcePathStyle())
                                .requestChecksumCalculation(config.getRequestChecksumValidation())
@@ -254,15 +254,15 @@ public static S3Client makeSyncClient(TestConfig config, Region region, AwsCrede
 
     public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) {
         switch (config.getFlavor()) {
-            case ASYNC_JAVA_BASED:
+            case STANDARD_ASYNC:
                 return S3AsyncClient.builder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())
                                     .region(region)
                                     .credentialsProvider(provider)
                                     .accelerate(config.isAccelerateEnabled())
                                     .build();
-            case TM_JAVA:
+            case MULTIPART_ENABLED:
                 return S3AsyncClient.builder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())
@@ -271,7 +271,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw
                                     .accelerate(config.isAccelerateEnabled())
                                     .multipartEnabled(true)
                                     .build();
-            case ASYNC_CRT: {
+            case CRT_BASED: {
                 return S3AsyncClient.crtBuilder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())
@@ -288,7 +288,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw
     public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
                                    Region region, AwsCredentialsProvider provider) {
         switch (config.getFlavor()) {
-            case JAVA_BASED:
+            case STANDARD_SYNC:
                 return S3Client.builder()
                                .overrideConfiguration(overrideConfiguration)
                                .forcePathStyle(config.isForcePathStyle())
@@ -305,7 +305,7 @@ public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfigura
     public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
                                          Region region, AwsCredentialsProvider provider) {
         switch (config.getFlavor()) {
-            case ASYNC_JAVA_BASED:
+            case STANDARD_ASYNC:
                 return S3AsyncClient.builder()
                                     .overrideConfiguration(overrideConfiguration)
                                     .forcePathStyle(config.isForcePathStyle())
@@ -314,7 +314,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon
                                     .credentialsProvider(provider)
                                     .accelerate(config.isAccelerateEnabled())
                                     .build();
-            case TM_JAVA:
+            case MULTIPART_ENABLED:
                 return S3AsyncClient.builder()
                                     .overrideConfiguration(overrideConfiguration)
                                     .forcePathStyle(config.isForcePathStyle())
@@ -324,7 +324,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon
                                     .accelerate(config.isAccelerateEnabled())
                                     .multipartEnabled(true)
                                     .build();
-            case ASYNC_CRT: {
+            case CRT_BASED: {
                 return S3AsyncClient.crtBuilder()
                                     .forcePathStyle(config.isForcePathStyle())
                                     .requestChecksumCalculation(config.getRequestChecksumValidation())

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java

@@ -16,10 +16,10 @@
 package software.amazon.awssdk.services.s3.regression;
 
 public enum S3ClientFlavor {
-    JAVA_BASED(false),
-    ASYNC_JAVA_BASED(true),
-    TM_JAVA(true),
-    ASYNC_CRT(true)
+    STANDARD_SYNC(false),
+    STANDARD_ASYNC(true),
+    MULTIPART_ENABLED(true),
+    CRT_BASED(true)
     ;
 
     private final boolean async;