Skip to content

Commit 83731b2

Browse files
trivikrtrivikr
committed
chore(middleware-flexible-checksums): perform checksum calculation and validation by default
1 parent b9f75f7 commit 83731b2

9 files changed

+545
-172
lines changed

‎clients/client-s3/test/unit/flexibleChecksums.spec.ts

Lines changed: 198 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import { ChecksumAlgorithm } from "@aws-sdk/middleware-flexible-checksums";
1+
import {
2+
ChecksumAlgorithm,
3+
DEFAULT_CHECKSUM_ALGORITHM,
4+
RequestChecksumCalculation,
5+
ResponseChecksumValidation,
6+
} from "@aws-sdk/middleware-flexible-checksums";
27
import { HttpRequest } from "@smithy/protocol-http";
38
import { BuildMiddleware } from "@smithy/types";
49
import { Readable } from "stream";
@@ -7,7 +12,7 @@ import { describe, expect, test as it } from "vitest";
712
import { ChecksumAlgorithm as Algo, S3 } from "../../src/index";
813

914
describe("Flexible Checksums", () => {
10-
const testCases = [
15+
const testCases: [string, string | undefined, string][] = [
1116
["", ChecksumAlgorithm.CRC32, "AAAAAA=="],
1217
["abc", ChecksumAlgorithm.CRC32, "NSRBwg=="],
1318
["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="],
@@ -23,148 +28,204 @@ describe("Flexible Checksums", () => {
2328
["", ChecksumAlgorithm.SHA256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="],
2429
["abc", ChecksumAlgorithm.SHA256, "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="],
2530
["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="],
31+
32+
// Choose default checksum algorithm when explicily not provided.
33+
["Hello world", undefined, "i9aeUg=="],
2634
];
2735

2836
describe("putObject", () => {
29-
testCases.forEach(([body, checksumAlgorithm, checksumValue]) => {
30-
const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`;
31-
32-
describe(`sets ${checksumHeader}="${checksumValue}"" for checksum="${checksumAlgorithm}"`, () => {
33-
const getBodyAsReadableStream = (content: string) => {
34-
const readableStream = new Readable();
35-
const separator = " ";
36-
const wordsAsChunks = content.split(separator);
37-
wordsAsChunks.forEach((word, index) => {
38-
readableStream.push(word);
39-
if (index !== wordsAsChunks.length - 1) {
40-
readableStream.push(separator);
41-
}
42-
});
43-
readableStream.push(null);
44-
return readableStream;
45-
};
46-
47-
it(`when body is sent as a request`, async () => {
48-
const requestChecksumValidator: BuildMiddleware<any, any> = (next) => async (args) => {
49-
// middleware intercept the request and return it early
50-
const request = args.request as HttpRequest;
51-
const { headers } = request;
52-
expect(headers["x-amz-sdk-checksum-algorithm"]).to.equal(checksumAlgorithm);
53-
expect(headers[checksumHeader]).to.equal(checksumValue);
54-
return { output: {} as any, response: {} as any };
55-
};
56-
57-
const client = new S3({
58-
region: "us-west-2",
59-
credentials: {
60-
accessKeyId: "CLIENT_TEST",
61-
secretAccessKey: "CLIENT_TEST",
62-
},
63-
});
64-
client.middlewareStack.addRelativeTo(requestChecksumValidator, {
65-
relation: "after",
66-
toMiddleware: "flexibleChecksumsMiddleware",
67-
});
68-
69-
return await client.putObject({
70-
Bucket: "bucket",
71-
Key: "key",
72-
Body: body,
73-
ChecksumAlgorithm: checksumAlgorithm as Algo,
74-
});
75-
});
76-
77-
it(`when body is sent as a stream`, async () => {
78-
const requestChecksumValidator: BuildMiddleware<any, any> = (next) => async (args) => {
79-
// middleware intercept the request and return it early
80-
const request = args.request as HttpRequest;
81-
const { headers, body } = request;
82-
expect(headers["content-length"]).to.be.undefined;
83-
expect(headers["content-encoding"]).to.equal("aws-chunked");
84-
expect(headers["transfer-encoding"]).to.equal("chunked");
85-
expect(headers["x-amz-content-sha256"]).to.equal("STREAMING-UNSIGNED-PAYLOAD-TRAILER");
86-
expect(headers["x-amz-trailer"]).to.equal(checksumHeader);
87-
body.on("data", (data: any) => {
88-
const stringValue = data.toString();
89-
if (stringValue.startsWith(checksumHeader)) {
90-
const receivedChecksum = stringValue.replace("\r\n", "").split(":")[1];
91-
expect(receivedChecksum).to.equal(checksumValue);
92-
}
37+
describe.each([undefined, RequestChecksumCalculation.WHEN_SUPPORTED, RequestChecksumCalculation.WHEN_REQUIRED])(
38+
`when requestChecksumCalculation='%s'`,
39+
(requestChecksumCalculation) => {
40+
describe.each(testCases)(
41+
`for body="%s" and checksumAlgorithm="%s", sets checksum="%s"`,
42+
(body, checksumAlgorithm, checksumValue) => {
43+
const checksumHeader = `x-amz-checksum-${(checksumAlgorithm ?? DEFAULT_CHECKSUM_ALGORITHM).toLowerCase()}`;
44+
const getBodyAsReadableStream = (content: string) => {
45+
const readableStream = new Readable();
46+
const separator = " ";
47+
const wordsAsChunks = content.split(separator);
48+
wordsAsChunks.forEach((word, index) => {
49+
readableStream.push(word);
50+
if (index !== wordsAsChunks.length - 1) {
51+
readableStream.push(separator);
52+
}
53+
});
54+
readableStream.push(null);
55+
return readableStream;
56+
};
57+
58+
it(`when body is sent as a string`, async () => {
59+
const requestChecksumValidator: BuildMiddleware<any, any> = (next) => async (args) => {
60+
// middleware intercept the request and return it early
61+
const request = args.request as HttpRequest;
62+
const { headers } = request;
63+
64+
// Headers are not set when checksumAlgorithm is not provided,
65+
// and requestChecksumCalculation is explicitly set to WHEN_SUPPORTED.
66+
if (
67+
checksumAlgorithm === undefined &&
68+
requestChecksumCalculation === RequestChecksumCalculation.WHEN_REQUIRED
69+
) {
70+
expect(headers["x-amz-sdk-checksum-algorithm"]).toBeUndefined();
71+
expect(headers[checksumHeader]).toBeUndefined();
72+
} else {
73+
expect(headers["x-amz-sdk-checksum-algorithm"]).toEqual(
74+
checksumAlgorithm ?? DEFAULT_CHECKSUM_ALGORITHM
75+
);
76+
expect(headers[checksumHeader]).toEqual(checksumValue);
77+
}
78+
79+
return { output: {} as any, response: {} as any };
80+
};
81+
82+
const client = new S3({
83+
region: "us-west-2",
84+
credentials: {
85+
accessKeyId: "CLIENT_TEST",
86+
secretAccessKey: "CLIENT_TEST",
87+
},
88+
requestChecksumCalculation,
89+
});
90+
client.middlewareStack.addRelativeTo(requestChecksumValidator, {
91+
relation: "after",
92+
toMiddleware: "flexibleChecksumsMiddleware",
93+
});
94+
95+
return await client.putObject({
96+
Bucket: "bucket",
97+
Key: "key",
98+
Body: body,
99+
ChecksumAlgorithm: checksumAlgorithm as Algo,
100+
});
101+
});
102+
103+
it(`when body is sent as a stream`, async () => {
104+
const requestChecksumValidator: BuildMiddleware<any, any> = (next) => async (args) => {
105+
// middleware intercept the request and return it early
106+
const request = args.request as HttpRequest;
107+
const { headers, body } = request;
108+
expect(headers["content-length"]).toBeUndefined();
109+
110+
// Headers are not set when checksumAlgorithm is not provided,
111+
// and requestChecksumCalculation is explicitly set to WHEN_SUPPORTED.
112+
if (
113+
checksumAlgorithm === undefined &&
114+
requestChecksumCalculation === RequestChecksumCalculation.WHEN_REQUIRED
115+
) {
116+
expect(headers["content-encoding"]).toBeUndefined();
117+
expect(headers["transfer-encoding"]).toBeUndefined();
118+
expect(headers["x-amz-content-sha256"]).toBeUndefined();
119+
expect(headers["x-amz-trailer"]).toBeUndefined();
120+
} else {
121+
expect(headers["content-encoding"]).toEqual("aws-chunked");
122+
expect(headers["transfer-encoding"]).toEqual("chunked");
123+
expect(headers["x-amz-content-sha256"]).toEqual("STREAMING-UNSIGNED-PAYLOAD-TRAILER");
124+
expect(headers["x-amz-trailer"]).toEqual(checksumHeader);
125+
}
126+
body.on("data", (data: any) => {
127+
const stringValue = data.toString();
128+
if (stringValue.startsWith(checksumHeader)) {
129+
const receivedChecksum = stringValue.replace("\r\n", "").split(":")[1];
130+
expect(receivedChecksum).toEqual(checksumValue);
131+
}
132+
});
133+
return { output: {} as any, response: {} as any };
134+
};
135+
136+
const client = new S3({
137+
region: "us-west-2",
138+
credentials: {
139+
accessKeyId: "CLIENT_TEST",
140+
secretAccessKey: "CLIENT_TEST",
141+
},
142+
requestChecksumCalculation,
143+
});
144+
client.middlewareStack.addRelativeTo(requestChecksumValidator, {
145+
relation: "after",
146+
toMiddleware: "flexibleChecksumsMiddleware",
147+
});
148+
149+
const bodyStream = getBodyAsReadableStream(body);
150+
await client.putObject({
151+
Bucket: "bucket",
152+
Key: "key",
153+
Body: bodyStream,
154+
ChecksumAlgorithm: checksumAlgorithm as Algo,
155+
});
93156
});
94-
return { output: {} as any, response: {} as any };
95-
};
96-
97-
const client = new S3({
98-
region: "us-west-2",
99-
credentials: {
100-
accessKeyId: "CLIENT_TEST",
101-
secretAccessKey: "CLIENT_TEST",
102-
},
103-
});
104-
client.middlewareStack.addRelativeTo(requestChecksumValidator, {
105-
relation: "after",
106-
toMiddleware: "flexibleChecksumsMiddleware",
107-
});
108-
109-
const bodyStream = getBodyAsReadableStream(body);
110-
await client.putObject({
111-
Bucket: "bucket",
112-
Key: "key",
113-
Body: bodyStream,
114-
ChecksumAlgorithm: checksumAlgorithm as Algo,
115-
});
116-
});
117-
});
118-
});
157+
}
158+
);
159+
}
160+
);
119161
});
120162

121163
describe("getObject", async () => {
122-
testCases.forEach(([body, checksumAlgorithm, checksumValue]) => {
123-
const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`;
124-
125-
it(`validates ${checksumHeader}="${checksumValue}"" set for checksum="${checksumAlgorithm}"`, async () => {
126-
const responseBody = new Readable();
127-
responseBody.push(body);
128-
responseBody.push(null);
129-
const responseChecksumValidator: BuildMiddleware<any, any> = (next, context) => async (args) => {
130-
const request = args.request as HttpRequest;
131-
return {
132-
output: {
133-
$metadata: { attempts: 0, httpStatusCode: 200 },
134-
request,
135-
context,
136-
Body: responseBody,
137-
} as any,
138-
response: {
139-
body: responseBody,
140-
headers: {
141-
[checksumHeader]: checksumValue,
164+
describe.each([undefined, ResponseChecksumValidation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED])(
165+
`when responseChecksumValidation='%s'`,
166+
(responseChecksumValidation) => {
167+
it.each(testCases)(
168+
`for body="%s" and checksumAlgorithm="%s", validates ChecksumMode`,
169+
async (body, checksumAlgorithm, checksumValue) => {
170+
const checksumHeader = `x-amz-checksum-${(checksumAlgorithm ?? DEFAULT_CHECKSUM_ALGORITHM).toLowerCase()}`;
171+
172+
const responseBody = new Readable();
173+
responseBody.push(body);
174+
responseBody.push(null);
175+
const responseChecksumValidator: BuildMiddleware<any, any> = (next, context) => async (args) => {
176+
// ChecksumMode is not set when checksumAlgorithm is not provided,
177+
// and responseChecksumValidation is explicitly set to WHEN_SUPPORTED.
178+
if (
179+
checksumAlgorithm === undefined &&
180+
responseChecksumValidation === ResponseChecksumValidation.WHEN_REQUIRED
181+
) {
182+
expect(args.input.ChecksumMode).toBeUndefined();
183+
} else {
184+
expect(args.input.ChecksumMode).toEqual("ENABLED");
185+
}
186+
187+
const request = args.request as HttpRequest;
188+
return {
189+
output: {
190+
$metadata: { attempts: 0, httpStatusCode: 200 },
191+
request,
192+
context,
193+
Body: responseBody,
194+
} as any,
195+
response: {
196+
body: responseBody,
197+
headers: {
198+
[checksumHeader]: checksumValue,
199+
},
200+
} as any,
201+
};
202+
};
203+
204+
const client = new S3({
205+
region: "us-west-2",
206+
credentials: {
207+
accessKeyId: "CLIENT_TEST",
208+
secretAccessKey: "CLIENT_TEST",
142209
},
143-
} as any,
144-
};
145-
};
146-
147-
const client = new S3({
148-
region: "us-west-2",
149-
credentials: {
150-
accessKeyId: "CLIENT_TEST",
151-
secretAccessKey: "CLIENT_TEST",
152-
},
153-
});
154-
client.middlewareStack.addRelativeTo(responseChecksumValidator, {
155-
relation: "after",
156-
toMiddleware: "flexibleChecksumsMiddleware",
157-
});
158-
159-
const { Body } = await client.getObject({
160-
Bucket: "bucket",
161-
Key: "key",
162-
ChecksumMode: "ENABLED",
163-
});
164-
(Body as Readable).on("data", (chunk) => {
165-
expect(chunk.toString()).to.equal(body);
166-
});
167-
});
168-
});
210+
responseChecksumValidation,
211+
});
212+
client.middlewareStack.addRelativeTo(responseChecksumValidator, {
213+
relation: "after",
214+
toMiddleware: "flexibleChecksumsMiddleware",
215+
});
216+
217+
const { Body } = await client.getObject({
218+
Bucket: "bucket",
219+
Key: "key",
220+
// Do not pass ChecksumMode if algorithm is not explicitly defined. It'll be set by SDK.
221+
ChecksumMode: checksumAlgorithm ? "ENABLED" : undefined,
222+
});
223+
(Body as Readable).on("data", (chunk) => {
224+
expect(chunk.toString()).toEqual(body);
225+
});
226+
}
227+
);
228+
}
229+
);
169230
});
170231
});

‎packages/middleware-flexible-checksums/src/configuration.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ import {
44
Encoder,
55
GetAwsChunkedEncodingStream,
66
HashConstructor,
7+
Provider,
78
StreamCollector,
89
StreamHasher,
910
} from "@smithy/types";
1011

12+
import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
13+
1114
export interface PreviouslyResolved {
1215
/**
1316
* The function that will be used to convert binary data to a base64-encoded string.
@@ -31,6 +34,16 @@ export interface PreviouslyResolved {
3134
*/
3235
md5: ChecksumConstructor | HashConstructor;
3336

37+
/**
38+
* Determines when a checksum will be calculated for request payloads
39+
*/
40+
requestChecksumCalculation: Provider<RequestChecksumCalculation>;
41+
42+
/**
43+
* Determines when a checksum will be calculated for response payloads
44+
*/
45+
responseChecksumValidation: Provider<ResponseChecksumValidation>;
46+
3447
/**
3548
* A constructor for a class implementing the {@link Hash} interface that computes SHA1 hashes.
3649
* @internal

0 commit comments

Comments
 (0)