Skip to content

Various performance improvements. #1791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 24, 2020
Merged

Various performance improvements. #1791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/next-release/feature-AWSSDKforJavav2-49d3cd6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "feature",
"category": "AWS SDK for Java v2",
"description": "Various performance improvements."
}
86 changes: 47 additions & 39 deletions core/auth/src/main/java/software/amazon/awssdk/auth/signer/internal/AbstractAws4Signer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
Expand Down Expand Up @@ -75,7 +76,14 @@ protected SdkHttpFullRequest.Builder doSign(SdkHttpFullRequest request,
.filter(h -> h.equals("required"))
.ifPresent(h -> mutableRequest.putHeader(SignerConstant.X_AMZ_CONTENT_SHA256, contentSha256));

String canonicalRequest = createCanonicalRequest(mutableRequest, contentSha256, signingParams.doubleUrlEncode());
Map<String, List<String>> canonicalHeaders = canonicalizeSigningHeaders(mutableRequest.headers());
String signedHeadersString = getSignedHeadersString(canonicalHeaders);

String canonicalRequest = createCanonicalRequest(mutableRequest,
canonicalHeaders,
signedHeadersString,
contentSha256,
signingParams.doubleUrlEncode());

String stringToSign = createStringToSign(canonicalRequest, requestParams);

Expand All @@ -84,7 +92,7 @@ protected SdkHttpFullRequest.Builder doSign(SdkHttpFullRequest request,
byte[] signature = computeSignature(stringToSign, signingKey);

mutableRequest.putHeader(SignerConstant.AUTHORIZATION,
buildAuthorizationHeader(signature, sanitizedCredentials, requestParams, mutableRequest));
buildAuthorizationHeader(signature, sanitizedCredentials, requestParams, signedHeadersString));

processRequestPayload(mutableRequest, signature, signingKey, requestParams, signingParams);

Expand All @@ -110,11 +118,16 @@ protected SdkHttpFullRequest.Builder doPresign(SdkHttpFullRequest request,
}

// Add the important parameters for v4 signing
addPreSignInformationToRequest(mutableRequest, sanitizedCredentials, requestParams, expirationInSeconds);
Map<String, List<String>> canonicalizedHeaders = canonicalizeSigningHeaders(mutableRequest.headers());
String signedHeadersString = getSignedHeadersString(canonicalizedHeaders);

addPreSignInformationToRequest(mutableRequest, signedHeadersString, sanitizedCredentials,
requestParams, expirationInSeconds);

String contentSha256 = calculateContentHashPresign(mutableRequest, signingParams);

String canonicalRequest = createCanonicalRequest(mutableRequest, contentSha256, signingParams.doubleUrlEncode());
String canonicalRequest = createCanonicalRequest(mutableRequest, canonicalizedHeaders, signedHeadersString,
contentSha256, signingParams.doubleUrlEncode());

String stringToSign = createStringToSign(canonicalRequest, requestParams);

Expand Down Expand Up @@ -191,19 +204,20 @@ protected final byte[] deriveSigningKey(AwsCredentials credentials, Instant sign
* generate the canonical request.
*/
private String createCanonicalRequest(SdkHttpFullRequest.Builder request,
Map<String, List<String>> canonicalHeaders,
String signedHeadersString,
String contentSha256,
boolean doubleUrlEncode) {

String canonicalRequest = request.method().toString() +
SignerConstant.LINE_SEPARATOR +
// This would optionally double url-encode the resource path
getCanonicalizedResourcePath(request.encodedPath(), doubleUrlEncode) +
SignerConstant.LINE_SEPARATOR +
getCanonicalizedQueryString(request.rawQueryParameters()) +
SignerConstant.LINE_SEPARATOR +
getCanonicalizedHeaderString(request.headers()) +
getCanonicalizedHeaderString(canonicalHeaders) +
SignerConstant.LINE_SEPARATOR +
getSignedHeadersString(request.headers()) +
signedHeadersString +
SignerConstant.LINE_SEPARATOR +
contentSha256;

Expand Down Expand Up @@ -254,12 +268,11 @@ private byte[] computeSignature(String stringToSign, byte[] signingKey) {
private String buildAuthorizationHeader(byte[] signature,
AwsCredentials credentials,
Aws4SignerRequestParams signerParams,
SdkHttpFullRequest.Builder mutableRequest) {
String signedHeadersString) {

String signingCredentials = credentials.accessKeyId() + "/" + signerParams.getScope();
String credential = "Credential=" + signingCredentials;
String signerHeaders = "SignedHeaders=" +
getSignedHeadersString(mutableRequest.headers());
String signerHeaders = "SignedHeaders=" + signedHeadersString;
String signatureHeader = "Signature=" + BinaryUtils.toHex(signature);

return SignerConstant.AWS4_SIGNING_ALGORITHM + " " + credential + ", " + signerHeaders + ", " + signatureHeader;
Expand All @@ -269,6 +282,7 @@ private String buildAuthorizationHeader(byte[] signature,
* Includes all the signing headers as request parameters for pre-signing.
*/
private void addPreSignInformationToRequest(SdkHttpFullRequest.Builder mutableRequest,
String signedHeadersString,
AwsCredentials sanitizedCredentials,
Aws4SignerRequestParams signerParams,
long expirationInSeconds) {
Expand All @@ -277,34 +291,39 @@ private void addPreSignInformationToRequest(SdkHttpFullRequest.Builder mutableRe

mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_ALGORITHM, SignerConstant.AWS4_SIGNING_ALGORITHM);
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_DATE, signerParams.getFormattedRequestSigningDateTime());
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_SIGNED_HEADER,
getSignedHeadersString(mutableRequest.headers()));
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_EXPIRES,
Long.toString(expirationInSeconds));
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_SIGNED_HEADER, signedHeadersString);
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_EXPIRES, Long.toString(expirationInSeconds));
mutableRequest.putRawQueryParameter(SignerConstant.X_AMZ_CREDENTIAL, signingCredentials);
}

private Map<String, List<String>> canonicalizeSigningHeaders(Map<String, List<String>> headers) {
Map<String, List<String>> result = new TreeMap<>();

private String getCanonicalizedHeaderString(Map<String, List<String>> headers) {
List<String> sortedHeaders = new ArrayList<>(headers.keySet());
sortedHeaders.sort(String.CASE_INSENSITIVE_ORDER);

StringBuilder buffer = new StringBuilder();
for (String header : sortedHeaders) {
if (shouldExcludeHeaderFromSigning(header)) {
for (Map.Entry<String, List<String>> header : headers.entrySet()) {
String lowerCaseHeader = lowerCase(header.getKey());
if (LIST_OF_HEADERS_TO_IGNORE_IN_LOWER_CASE.contains(lowerCaseHeader)) {
continue;
}
String key = lowerCase(header);

for (String headerValue : headers.get(header)) {
appendCompactedString(buffer, key);
result.computeIfAbsent(lowerCaseHeader, x -> new ArrayList<>()).addAll(header.getValue());
}

return result;
}

private String getCanonicalizedHeaderString(Map<String, List<String>> canonicalizedHeaders) {
StringBuilder buffer = new StringBuilder();

canonicalizedHeaders.forEach((headerName, headerValues) -> {
for (String headerValue : headerValues) {
appendCompactedString(buffer, headerName);
buffer.append(":");
if (headerValue != null) {
appendCompactedString(buffer, headerValue);
}
buffer.append("\n");
}
}
});

return buffer.toString();
}
Expand Down Expand Up @@ -350,28 +369,17 @@ private boolean isWhiteSpace(final char ch) {
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\u000b' || ch == '\r' || ch == '\f';
}

private String getSignedHeadersString(Map<String, List<String>> headers) {
List<String> sortedHeaders = new ArrayList<>(headers.keySet());
sortedHeaders.sort(String.CASE_INSENSITIVE_ORDER);

private String getSignedHeadersString(Map<String, List<String>> canonicalizedHeaders) {
StringBuilder buffer = new StringBuilder();
for (String header : sortedHeaders) {
if (shouldExcludeHeaderFromSigning(header)) {
continue;
}
for (String header : canonicalizedHeaders.keySet()) {
if (buffer.length() > 0) {
buffer.append(";");
}
buffer.append(lowerCase(header));
buffer.append(header);
}

return buffer.toString();
}

private boolean shouldExcludeHeaderFromSigning(String header) {
return LIST_OF_HEADERS_TO_IGNORE_IN_LOWER_CASE.contains(lowerCase(header));
}

private void addHostHeader(SdkHttpFullRequest.Builder mutableRequest) {
// AWS4 requires that we sign the Host header so we
// have to have it in the request by the time we sign.
Expand Down
6 changes: 3 additions & 3 deletions core/sdk-core/src/main/java/software/amazon/awssdk/core/RequestOverrideConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ default B putRawQueryParameter(String name, String value) {
}

protected abstract static class BuilderImpl<B extends Builder> implements Builder<B> {
private Map<String, List<String>> headers = new HashMap<>();
private Map<String, List<String>> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
private Map<String, List<String>> rawQueryParameters = new HashMap<>();
private List<ApiName> apiNames = new ArrayList<>();
private Duration apiCallTimeout;
Expand All @@ -366,7 +366,7 @@ protected BuilderImpl(RequestOverrideConfiguration sdkRequestOverrideConfig) {

@Override
public Map<String, List<String>> headers() {
return CollectionUtils.deepUnmodifiableMap(headers, () -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER));
return CollectionUtils.unmodifiableMapOfLists(headers);
}

@Override
Expand All @@ -386,7 +386,7 @@ public B headers(Map<String, List<String>> headers) {

@Override
public Map<String, List<String>> rawQueryParameters() {
return CollectionUtils.deepUnmodifiableMap(rawQueryParameters);
return CollectionUtils.unmodifiableMapOfLists(rawQueryParameters);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion .../src/main/java/software/amazon/awssdk/core/client/config/ClientOverrideConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ public void setHeaders(Map<String, List<String>> additionalHttpHeaders) {

@Override
public Map<String, List<String>> headers() {
return CollectionUtils.deepUnmodifiableMap(headers);
return CollectionUtils.unmodifiableMapOfLists(headers);
}

@Override
Expand Down
10 changes: 4 additions & 6 deletions ...software/amazon/awssdk/core/internal/http/pipeline/stages/utils/RetryableStageHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,11 @@ public SdkHttpFullRequest requestToSend() {
Integer availableRetryCapacity = TokenBucketRetryCondition.getCapacityForExecution(context.executionAttributes())
.map(TokenBucketRetryCondition.Capacity::capacityRemaining)
.orElse(null);

String headerValue = (attemptNumber - 1) + "/" +
lastBackoffDelay.toMillis() + "/" +
(availableRetryCapacity != null ? availableRetryCapacity : "");
return request.toBuilder()
.putHeader(SDK_RETRY_INFO_HEADER,
String.format("%s/%s/%s",
attemptNumber - 1,
lastBackoffDelay.toMillis(),
availableRetryCapacity != null ? availableRetryCapacity : ""))
.putHeader(SDK_RETRY_INFO_HEADER, headerValue)
.build();
}

Expand Down
Loading