Skip to content

Commit a7e4a94

Browse files
zoewanggzoewangg
committed
Refactor s3-benchmarks and include v1 transfermanager tests
1 parent 081679c commit a7e4a94

14 files changed

+448
-83
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@
112112
<rxjava.version>2.2.21</rxjava.version>
113113
<commons-codec.verion>1.10</commons-codec.verion>
114114
<jmh.version>1.29</jmh.version>
115-
<awscrt.version>0.13.2</awscrt.version>
115+
<awscrt.version>0.13.5</awscrt.version>
116116

117117
<!--Test dependencies -->
118118
<junit.version>4.13.1</junit.version>

‎test/s3-benchmarks/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# S3 Benchmark Harness
22

33

4-
This module contains performance tests for `S3AsyncClient` and
4+
This module contains performance tests for `S3AsyncClient` and
55
`S3TransferManager`
66

77
## How to run
@@ -11,8 +11,8 @@ This module contains performance tests for `S3AsyncClient` and
1111
mvn clean install -pl :s3-benchmarks -P quick --am
1212
1313
# download
14-
java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/destionfile/ --operation=download --partSizeInMB=20 --maxThroughput=100.0
14+
java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/destionfile/ --operation=download --partSizeInMB=20 --maxThroughput=100.0 --version=v2 --warmupConcurrency=300
1515
1616
# upload
17-
java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/sourcefile/ --operation=upload --partSizeInMB=20 --maxThroughput=100.0
17+
java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/sourcefile/ --operation=upload --partSizeInMB=20 --maxThroughput=100.0 --version=v2 --warmupConcurrency=300
1818
```

‎test/s3-benchmarks/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
<maven.compiler.source>8</maven.compiler.source>
3131
<maven.compiler.target>8</maven.compiler.target>
3232
<awsjavasdk.version>${project.version}</awsjavasdk.version>
33+
<sdk-v1.version>1.12.1</sdk-v1.version>
3334
</properties>
3435
<name>AWS Java SDK :: Test :: S3 Benchmarks</name>
3536
<description>Contains benchmark code for S3 and TransferManager</description>
@@ -55,6 +56,11 @@
5556
<artifactId>s3-transfer-manager</artifactId>
5657
<version>${awsjavasdk.version}-PREVIEW</version>
5758
</dependency>
59+
<dependency>
60+
<groupId>com.amazonaws</groupId>
61+
<artifactId>aws-java-sdk-s3</artifactId>
62+
<version>${sdk-v1.version}</version>
63+
</dependency>
5864
<dependency>
5965
<groupId>software.amazon.awssdk</groupId>
6066
<artifactId>test-utils</artifactId>

‎test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BaseTransferManagerBenchmark.java

Lines changed: 15 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515

1616
package software.amazon.awssdk.s3benchmarks;
1717

18+
import static software.amazon.awssdk.s3benchmarks.BenchmarkUtils.DEFAULT_WARMUP_CONCURRENCY;
19+
import static software.amazon.awssdk.s3benchmarks.BenchmarkUtils.PRE_WARMUP_ITERATIONS;
20+
import static software.amazon.awssdk.s3benchmarks.BenchmarkUtils.PRE_WARMUP_RUNS;
21+
import static software.amazon.awssdk.transfer.s3.SizeConstant.KB;
22+
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;
1823
import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError;
1924

2025
import java.io.File;
@@ -28,46 +33,39 @@
2833
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
2934
import software.amazon.awssdk.services.s3.S3Client;
3035
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
31-
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
3236
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
3337
import software.amazon.awssdk.testutils.RandomTempFile;
34-
import software.amazon.awssdk.transfer.s3.S3TransferManager;
3538
import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient;
3639
import software.amazon.awssdk.utils.Logger;
3740

3841
public abstract class BaseTransferManagerBenchmark implements TransferManagerBenchmark {
39-
protected static final int WARMUP_ITERATIONS = 10;
40-
protected static final int BENCHMARK_ITERATIONS = 10;
4142

4243
private static final Logger logger = Logger.loggerFor("TransferManagerBenchmark");
4344
private static final String WARMUP_KEY = "warmupobject";
4445

45-
protected final S3TransferManager transferManager;
4646
protected final S3CrtAsyncClient s3;
4747
protected final S3Client s3Sync;
4848
protected final String bucket;
4949
protected final String key;
5050
protected final String path;
51-
private final File file;
51+
protected final int warmupConcurrency;
52+
protected final File tmpFile;
5253

5354
BaseTransferManagerBenchmark(TransferManagerBenchmarkConfig config) {
5455
logger.info(() -> "Benchmark config: " + config);
55-
Long partSizeInMb = config.partSizeInMb() == null ? null : config.partSizeInMb() * 1024 * 1024L;
56+
Long partSizeInMb = config.partSizeInMb() == null ? null : config.partSizeInMb() * MB;
5657
s3 = S3CrtAsyncClient.builder()
5758
.targetThroughputInGbps(config.targetThroughput())
5859
.minimumPartSizeInBytes(partSizeInMb)
5960
.build();
6061
s3Sync = S3Client.builder()
6162
.build();
62-
transferManager = S3TransferManager.builder()
63-
.s3ClientConfiguration(b -> b.targetThroughputInGbps(config.targetThroughput())
64-
.minimumPartSizeInBytes(partSizeInMb))
65-
.build();
6663
bucket = config.bucket();
6764
key = config.key();
6865
path = config.filePath();
66+
warmupConcurrency = config.warmupConcurrency() == null ? DEFAULT_WARMUP_CONCURRENCY : config.warmupConcurrency();
6967
try {
70-
file = new RandomTempFile(1024 * 1000L);
68+
tmpFile = new RandomTempFile(1000 * KB);
7169
} catch (IOException e) {
7270
logger.error(() -> "Failed to create the file");
7371
throw new RuntimeException("Failed to create the temp file", e);
@@ -95,36 +93,15 @@ protected void additionalWarmup() {
9593

9694
protected abstract void doRunBenchmark();
9795

98-
protected final void printOutResult(List<Double> metrics, String name) {
99-
logger.info(() -> String.format("=============== %s Result ================", name));
100-
logger.info(() -> "" + metrics);
101-
double averageLatency = metrics.stream()
102-
.mapToDouble(a -> a)
103-
.average()
104-
.orElse(0.0);
105-
106-
double lowestLatency = metrics.stream()
107-
.mapToDouble(a -> a)
108-
.min().orElse(0.0);
109-
110-
HeadObjectResponse headObjectResponse = s3Sync.headObject(b -> b.bucket(bucket).key(key));
111-
double contentLengthInGigabit = (headObjectResponse.contentLength() / (1000 * 1000 * 1000.0)) * 8.0;
112-
logger.info(() -> "Average latency (s): " + averageLatency);
113-
logger.info(() -> "Object size (Gigabit): " + contentLengthInGigabit);
114-
logger.info(() -> "Average throughput (Gbps): " + contentLengthInGigabit / averageLatency);
115-
logger.info(() -> "Highest average throughput (Gbps): " + contentLengthInGigabit / lowestLatency);
116-
logger.info(() -> "==========================================================");
117-
}
118-
11996
private void cleanup() {
12097
s3Sync.deleteObject(b -> b.bucket(bucket).key(WARMUP_KEY));
121-
transferManager.close();
98+
s3.close();
12299
}
123100

124101
private void warmUp() throws InterruptedException {
125102
logger.info(() -> "Starting to warm up");
126103

127-
for (int i = 0; i < WARMUP_ITERATIONS; i++) {
104+
for (int i = 0; i < PRE_WARMUP_ITERATIONS; i++) {
128105
warmUpUploadBatch();
129106
warmUpDownloadBatch();
130107

@@ -136,7 +113,7 @@ private void warmUp() throws InterruptedException {
136113

137114
private void warmUpDownloadBatch() {
138115
List<CompletableFuture<?>> futures = new ArrayList<>();
139-
for (int i = 0; i < 20; i++) {
116+
for (int i = 0; i < PRE_WARMUP_RUNS; i++) {
140117
Path tempFile = RandomTempFile.randomUncreatedFile().toPath();
141118
futures.add(s3.getObject(GetObjectRequest.builder().bucket(bucket).key(WARMUP_KEY).build(),
142119
AsyncResponseTransformer.toFile(tempFile)).whenComplete((r, t) -> runAndLogError(
@@ -148,9 +125,9 @@ private void warmUpDownloadBatch() {
148125

149126
private void warmUpUploadBatch() {
150127
List<CompletableFuture<?>> futures = new ArrayList<>();
151-
for (int i = 0; i < 20; i++) {
128+
for (int i = 0; i < PRE_WARMUP_RUNS; i++) {
152129
futures.add(s3.putObject(PutObjectRequest.builder().bucket(bucket).key(WARMUP_KEY).build(),
153-
AsyncRequestBody.fromFile(file)));
130+
AsyncRequestBody.fromFile(tmpFile)));
154131
}
155132

156133
CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0])).join();

‎test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BenchmarkRunner.java

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ public class BenchmarkRunner {
2929
private static final String MAX_THROUGHPUT = "maxThroughput";
3030
private static final String KEY = "key";
3131
private static final String OPERATION = "operation";
32+
private static final String WARMUP_CONCURRENCY = "warmupConcurrency";
33+
private static final String VERSION = "version";
3234

3335
private BenchmarkRunner() {
3436
}
@@ -42,20 +44,35 @@ public static void main(String... args) throws org.apache.commons.cli.ParseExcep
4244
+ "uploaded");
4345
options.addRequiredOption(null, BUCKET, true, "The s3 bucket");
4446
options.addRequiredOption(null, KEY, true, "The s3 key");
45-
options.addRequiredOption(null, OPERATION, true, "The operation to benchmark against");
47+
options.addRequiredOption(null, OPERATION, true, "The operation to run tests: download | upload");
4648
options.addOption(null, PART_SIZE_IN_MB, true, "Part size in MB");
4749
options.addOption(null, MAX_THROUGHPUT, true, "The max throughput");
50+
options.addOption(null, WARMUP_CONCURRENCY, true, "The number of concurrent requests to send during warmup, default: "
51+
+ "100");
52+
options.addOption(null, VERSION, true, "The major version of the transfer manager to run test: v1 | v2, default: v2");
4853

4954
CommandLine cmd = parser.parse(options, args);
5055
TransferManagerBenchmarkConfig config = parseConfig(cmd);
5156
TransferManagerOperation operation = TransferManagerOperation.valueOf(cmd.getOptionValue(OPERATION)
5257
.toUpperCase(Locale.ENGLISH));
58+
59+
SdkVersion version = SdkVersion.valueOf(cmd.getOptionValue(VERSION, "V2")
60+
.toUpperCase(Locale.ENGLISH));
61+
5362
switch (operation) {
5463
case DOWNLOAD:
55-
TransferManagerBenchmark.download(config).run();
64+
if (version.equals(SdkVersion.V2)) {
65+
TransferManagerBenchmark.download(config).run();
66+
} else {
67+
TransferManagerBenchmark.v1Download(config).run();
68+
}
5669
break;
5770
case UPLOAD:
58-
TransferManagerBenchmark.upload(config).run();
71+
if (version.equals(SdkVersion.V2)) {
72+
TransferManagerBenchmark.upload(config).run();
73+
} else {
74+
TransferManagerBenchmark.v1Upload(config).run();
75+
}
5976
break;
6077
default:
6178
throw new UnsupportedOperationException();
@@ -72,9 +89,13 @@ private static TransferManagerBenchmarkConfig parseConfig(CommandLine cmd) {
7289
Double maxThroughput = cmd.getOptionValue(MAX_THROUGHPUT) == null ? null :
7390
Double.parseDouble(cmd.getOptionValue(MAX_THROUGHPUT));
7491

92+
Integer warmupConcurrency = cmd.getOptionValue(WARMUP_CONCURRENCY) == null ? null :
93+
Integer.parseInt(cmd.getOptionValue(WARMUP_CONCURRENCY));
94+
7595
return TransferManagerBenchmarkConfig.builder()
7696
.key(key)
7797
.bucket(bucket)
98+
.warmupConcurrency(warmupConcurrency)
7899
.partSizeInMb(partSize)
79100
.targetThroughput(maxThroughput)
80101
.filePath(filePath)
@@ -85,4 +106,9 @@ private enum TransferManagerOperation {
85106
DOWNLOAD,
86107
UPLOAD
87108
}
109+
110+
private enum SdkVersion {
111+
V1,
112+
V2
113+
}
88114
}

‎test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BenchmarkUtils.java

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.s3benchmarks;
17+
18+
import java.util.List;
19+
import software.amazon.awssdk.utils.Logger;
20+
21+
public class BenchmarkUtils {
22+
protected static final int PRE_WARMUP_ITERATIONS = 10;
23+
protected static final int PRE_WARMUP_RUNS = 20;
24+
protected static final int BENCHMARK_ITERATIONS = 10;
25+
protected static final int DEFAULT_WARMUP_CONCURRENCY = 100;
26+
protected static final String WARMUP_KEY = "warmupobject";
27+
28+
private static final Logger logger = Logger.loggerFor("TransferManagerBenchmark");
29+
30+
private BenchmarkUtils() {
31+
}
32+
33+
public static void printOutResult(List<Double> metrics, String name, long contentLengthInByte) {
34+
logger.info(() -> String.format("=============== %s Result ================", name));
35+
logger.info(() -> "" + metrics);
36+
double averageLatency = metrics.stream()
37+
.mapToDouble(a -> a)
38+
.average()
39+
.orElse(0.0);
40+
41+
double lowestLatency = metrics.stream()
42+
.mapToDouble(a -> a)
43+
.min().orElse(0.0);
44+
45+
double contentLengthInGigabit = (contentLengthInByte / (1000 * 1000 * 1000.0)) * 8.0;
46+
logger.info(() -> "Average latency (s): " + averageLatency);
47+
logger.info(() -> "Object size (Gigabit): " + contentLengthInGigabit);
48+
logger.info(() -> "Average throughput (Gbps): " + contentLengthInGigabit / averageLatency);
49+
logger.info(() -> "Highest average throughput (Gbps): " + contentLengthInGigabit / lowestLatency);
50+
logger.info(() -> "==========================================================");
51+
}
52+
}

‎test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmark.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,12 @@ static TransferManagerBenchmark upload(TransferManagerBenchmarkConfig config) {
3434
return new TransferManagerUploadBenchmark(config);
3535
}
3636

37+
static TransferManagerBenchmark v1Download(TransferManagerBenchmarkConfig config) {
38+
return new V1TransferManagerDownloadBenchmark(config);
39+
}
40+
41+
static TransferManagerBenchmark v1Upload(TransferManagerBenchmarkConfig config) {
42+
return new V1TransferManagerUploadBenchmark(config);
43+
}
44+
3745
}

‎test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmarkConfig.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ public class TransferManagerBenchmarkConfig {
2121
private final String key;
2222
private final Double targetThroughput;
2323
private final Long partSizeInMb;
24+
private final Integer warmupConcurrency;
2425

2526
private TransferManagerBenchmarkConfig(Builder builder) {
2627
this.filePath = builder.filePath;
2728
this.bucket = builder.bucket;
2829
this.key = builder.key;
2930
this.targetThroughput = builder.targetThroughput;
3031
this.partSizeInMb = builder.partSizeInMb;
32+
this.warmupConcurrency = builder.warmupConcurrency;
3133
}
3234

3335
public String filePath() {
@@ -50,6 +52,10 @@ public Long partSizeInMb() {
5052
return partSizeInMb;
5153
}
5254

55+
public Integer warmupConcurrency() {
56+
return warmupConcurrency;
57+
}
58+
5359
public static Builder builder() {
5460
return new Builder();
5561
}
@@ -61,7 +67,8 @@ public String toString() {
6167
", bucket: '" + bucket + '\'' +
6268
", key: '" + key + '\'' +
6369
", targetThroughput: " + targetThroughput +
64-
", partSizeInMB: " + partSizeInMb +
70+
", partSizeInMb: " + partSizeInMb +
71+
", warmupConcurrency: " + warmupConcurrency +
6572
'}';
6673
}
6774

@@ -71,6 +78,7 @@ static final class Builder {
7178
private String key;
7279
private Double targetThroughput;
7380
private Long partSizeInMb;
81+
private Integer warmupConcurrency;
7482

7583
public Builder filePath(String filePath) {
7684
this.filePath = filePath;
@@ -97,6 +105,11 @@ public Builder partSizeInMb(Long partSizeInMb) {
97105
return this;
98106
}
99107

108+
public Builder warmupConcurrency(Integer warmupConcurrency) {
109+
this.warmupConcurrency = warmupConcurrency;
110+
return this;
111+
}
112+
100113
public TransferManagerBenchmarkConfig build() {
101114
return new TransferManagerBenchmarkConfig(this);
102115
}

0 commit comments

Comments
 (0)