Skip to content

Commit 0084b1f

Browse files
L-ApplinL-Applin
committed
rename tp regression testing
seperate data-place, upload and download test avoid running useless tests cases
1 parent 78e6b5b commit 0084b1f

File tree

12 files changed

+790
-709
lines changed

12 files changed

+790
-709
lines changed

‎buildspecs/s3-checksums-tests.yml renamed to ‎buildspecs/s3-regression-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: 0.2
33
phases:
44
build:
55
commands:
6-
- mvn clean install -P s3-checksums-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS
6+
- mvn clean install -P s3-regression-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS
77
- echo $MAVEN_OPTIONS
88
finally:
99
- mkdir -p codebuild-test-reports

‎pom.xml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -862,7 +862,6 @@
862862
<include>**/*IntegrationTests.java</include>
863863
<include>**/*IntegTest.java</include>
864864
<include>**/RunCucumberTest.java</include>
865-
<include>**/ChecksumIntegrationTesting.java</include>
866865
</includes>
867866
<excludes>
868867
<exclude>**/SimpleMethodsIntegrationTest.java</exclude>
@@ -878,7 +877,7 @@
878877
</profile>
879878

880879
<profile>
881-
<id>s3-checksums-tests</id>
880+
<id>s3-regression-tests</id>
882881
<activation>
883882
<property>
884883
<name>doRelease</name>
@@ -911,8 +910,7 @@
911910
<!-- this test needs a lot of memory, we need to increase java heap size or we get OOM in codebuild -->
912911
<argLine>-Xmx12g -Xms4g -XX:+AllowRedefinitionToAddDeleteMethods</argLine>
913912
<includes>
914-
<!-- <include>**/s3/checksum/ChecksumIntegrationTesting.java</include>-->
915-
<include>**/s3/checksum/DownloadStreamingIntegrationTesting.java</include>
913+
<include>**/*RegressionTesting.java</include>
916914
</includes>
917915
<trimStackTrace>false</trimStackTrace>
918916
</configuration>

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BaseS3RegressionTest.java

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.regression;
17+
18+
import java.io.IOException;
19+
import java.util.ArrayList;
20+
import java.util.HashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
import org.junit.jupiter.api.AfterEach;
24+
import org.junit.jupiter.api.BeforeAll;
25+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
26+
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
27+
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
28+
import software.amazon.awssdk.regions.Region;
29+
import software.amazon.awssdk.services.s3.S3Client;
30+
import software.amazon.awssdk.services.s3control.S3ControlClient;
31+
import software.amazon.awssdk.services.sts.StsClient;
32+
import software.amazon.awssdk.utils.Logger;
33+
34+
public abstract class BaseS3RegressionTest {
35+
private static final Logger LOG = Logger.loggerFor(BaseS3RegressionTest.class);
36+
37+
private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-";
38+
private static final String MRAP_NAME = "do-not-delete-checksum-testing";
39+
private static final String AP_NAME = "do-not-delete-checksum-testing-ap";
40+
private static final String EOZ_SUFFIX = "--usw2-az3--x-s3";
41+
protected static final Region REGION = Region.US_WEST_2;
42+
43+
protected static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account";
44+
protected static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN =
45+
AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder()
46+
.profileName(TEST_CREDENTIALS_PROFILE_NAME)
47+
.build(),
48+
DefaultCredentialsProvider.create());
49+
50+
protected static String accountId;
51+
protected static String bucketName;
52+
protected static String mrapArn;
53+
protected static String eozBucket;
54+
protected static String apArn;
55+
56+
protected static S3ControlClient s3Control;
57+
protected static S3Client s3;
58+
protected static StsClient sts;
59+
60+
private Map<BucketType, List<String>> bucketCleanup = new HashMap<>();
61+
62+
@BeforeAll
63+
static void setup() throws InterruptedException, IOException {
64+
// Log.initLoggingToStdout(Log.LogLevel.Trace);
65+
66+
s3 = S3Client.builder()
67+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
68+
.region(REGION)
69+
.build();
70+
71+
s3Control = S3ControlClient.builder()
72+
.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
73+
.region(REGION)
74+
.build();
75+
76+
sts = StsClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
77+
.region(REGION)
78+
.build();
79+
80+
accountId = S3ChecksumsTestUtils.getAccountId(sts);
81+
bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG);
82+
mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG);
83+
eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG);
84+
apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName);
85+
86+
}
87+
88+
@AfterEach
89+
public void methodCleanup() {
90+
bucketCleanup.forEach((bt, keys) -> {
91+
String bucket = bucketForType(bt);
92+
keys.forEach(k -> s3.deleteObject(r -> r.bucket(bucket).key(k)));
93+
});
94+
95+
bucketCleanup.clear();
96+
}
97+
98+
protected void recordObjectToCleanup(BucketType type, String key) {
99+
bucketCleanup.computeIfAbsent(type, k -> new ArrayList<>()).add(key);
100+
}
101+
102+
protected static String getBucketName() {
103+
return BUCKET_NAME_PREFIX + accountId;
104+
}
105+
106+
protected static String bucketForType(BucketType type) {
107+
switch (type) {
108+
case STANDARD_BUCKET:
109+
return bucketName;
110+
case MRAP:
111+
return mrapArn;
112+
case EOZ:
113+
return eozBucket;
114+
case ACCESS_POINT:
115+
return apArn;
116+
default:
117+
throw new RuntimeException("Unknown bucket type: " + type);
118+
}
119+
}
120+
121+
}

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java renamed to ‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BucketType.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
* permissions and limitations under the License.
1414
*/
1515

16-
package software.amazon.awssdk.services.s3.checksum;
16+
package software.amazon.awssdk.services.s3.regression;
1717

1818
public enum BucketType {
1919
STANDARD_BUCKET(false),

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.regression;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType;
20+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz;
21+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle;
22+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle;
23+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32;
24+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeAsyncClient;
25+
import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeSyncClient;
26+
27+
import java.util.List;
28+
import java.util.concurrent.Callable;
29+
import org.junit.jupiter.api.Assumptions;
30+
import org.junit.jupiter.params.ParameterizedTest;
31+
import org.junit.jupiter.params.provider.MethodSource;
32+
import software.amazon.awssdk.awscore.AwsClient;
33+
import software.amazon.awssdk.core.sync.RequestBody;
34+
import software.amazon.awssdk.services.s3.S3AsyncClient;
35+
import software.amazon.awssdk.services.s3.S3Client;
36+
import software.amazon.awssdk.services.s3.model.Delete;
37+
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
38+
import software.amazon.awssdk.services.s3.model.GlacierJobParameters;
39+
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
40+
import software.amazon.awssdk.services.s3.model.RestoreObjectRequest;
41+
import software.amazon.awssdk.services.s3.model.RestoreRequest;
42+
import software.amazon.awssdk.services.s3.model.StorageClass;
43+
import software.amazon.awssdk.services.s3.model.Tier;
44+
import software.amazon.awssdk.utils.CompletableFutureUtils;
45+
import software.amazon.awssdk.utils.Logger;
46+
47+
public class DataplaneOperationRegressionTesting extends BaseS3RegressionTest {
48+
private static final Logger LOG = Logger.loggerFor(DataplaneOperationRegressionTesting.class);
49+
50+
// Request checksum required
51+
@ParameterizedTest
52+
@MethodSource("testConfigs")
53+
void deleteObject(TestConfig config) throws Exception {
54+
assumeNotAccessPointWithPathStyle(config);
55+
assumeNotAccelerateWithPathStyle(config);
56+
assumeNotAccelerateWithArnType(config);
57+
assumeNotAccelerateWithEoz(config);
58+
59+
LOG.debug(() -> "Running deleteObject with config: " + config.toString());
60+
61+
String bucket = bucketForType(config.getBucketType());
62+
String key = putRandomObject(config.getBucketType());
63+
TestCallable<Void> callable = null;
64+
try {
65+
DeleteObjectsRequest req = DeleteObjectsRequest.builder()
66+
.bucket(bucket)
67+
.delete(Delete.builder()
68+
.objects(ObjectIdentifier.builder()
69+
.key(key)
70+
.build())
71+
.build())
72+
.build();
73+
74+
callable = callDeleteObjects(req, config);
75+
callable.runnable().call();
76+
} finally {
77+
if (callable != null) {
78+
callable.client().close();
79+
}
80+
}
81+
}
82+
83+
// Request checksum optional
84+
@ParameterizedTest
85+
@MethodSource("testConfigs")
86+
void restoreObject(TestConfig config) throws Exception {
87+
assumeNotAccessPointWithPathStyle(config);
88+
assumeNotAccelerateWithPathStyle(config);
89+
assumeNotAccelerateWithArnType(config);
90+
91+
Assumptions.assumeFalse(config.getBucketType() == BucketType.EOZ,
92+
"Restore is not supported for S3 Express");
93+
94+
LOG.debug(() -> "Running restoreObject with config: " + config);
95+
96+
String bucket = bucketForType(config.getBucketType());
97+
String key = putRandomArchivedObject(config.getBucketType());
98+
TestCallable<Void> callable = null;
99+
try {
100+
RestoreObjectRequest request = RestoreObjectRequest.builder()
101+
.bucket(bucket)
102+
.key(key)
103+
.restoreRequest(RestoreRequest.builder()
104+
.days(5)
105+
.glacierJobParameters(GlacierJobParameters.builder()
106+
.tier(Tier.STANDARD)
107+
.build())
108+
.build())
109+
.build();
110+
111+
callable = callRestoreObject(request, config);
112+
callable.runnable().call();
113+
} finally {
114+
if (callable != null) {
115+
callable.client().close();
116+
}
117+
}
118+
}
119+
120+
private TestCallable<Void> callDeleteObjects(DeleteObjectsRequest request, TestConfig config) {
121+
AwsClient toClose;
122+
Callable<Void> runnable = null;
123+
124+
if (config.getFlavor().isAsync()) {
125+
S3AsyncClient s3Async = makeAsyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN);
126+
toClose = s3Async;
127+
runnable = () -> {
128+
CompletableFutureUtils.joinLikeSync(s3Async.deleteObjects(request));
129+
return null;
130+
};
131+
} else {
132+
S3Client s3 = makeSyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN);
133+
toClose = s3;
134+
runnable = () -> {
135+
s3.deleteObjects(request);
136+
return null;
137+
};
138+
}
139+
140+
return new TestCallable<>(toClose, runnable);
141+
}
142+
143+
private TestCallable<Void> callRestoreObject(RestoreObjectRequest request, TestConfig config) {
144+
AwsClient toClose;
145+
Callable<Void> callable = null;
146+
147+
if (config.getFlavor().isAsync()) {
148+
S3AsyncClient s3Async = makeAsyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN);
149+
toClose = s3Async;
150+
callable = () -> {
151+
s3Async.restoreObject(request).join();
152+
return null;
153+
};
154+
} else {
155+
S3Client s3 = makeSyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN);
156+
toClose = s3;
157+
callable = () -> {
158+
s3.restoreObject(request);
159+
return null;
160+
};
161+
}
162+
163+
return new TestCallable<>(toClose, callable);
164+
}
165+
166+
static List<TestConfig> testConfigs() {
167+
return TestConfig.testConfigs();
168+
}
169+
170+
private String putRandomObject(BucketType bucketType) {
171+
String key = S3ChecksumsTestUtils.randomKey();
172+
String bucketName = bucketForType(bucketType);
173+
s3.putObject(r -> r.bucket(bucketName).key(key), RequestBody.fromString("hello"));
174+
recordObjectToCleanup(bucketType, key);
175+
return key;
176+
}
177+
178+
private String putRandomArchivedObject(BucketType bucketType) {
179+
String key = S3ChecksumsTestUtils.randomKey();
180+
String bucketName = bucketForType(bucketType);
181+
s3.putObject(r -> r.bucket(bucketName).key(key).storageClass(StorageClass.GLACIER), RequestBody.fromString("hello"));
182+
recordObjectToCleanup(bucketType, key);
183+
return key;
184+
}
185+
186+
187+
}

0 commit comments

Comments
 (0)