test(middleware-flexible-checksums): add e2e test for md5 fallback

Author: siddsriv

packages/middleware-flexible-checksums/src/middleware-md5-fallback.e2e.spec.ts

@@ -0,0 +1,129 @@
+// see supplemental-docs/MD5_FALLBACK for more details
+import {
+  CreateBucketCommand,
+  DeleteBucketCommand,
+  DeleteObjectsCommand,
+  PutObjectCommand,
+  S3,
+} from "@aws-sdk/client-s3";
+import { createHash } from "crypto";
+import { afterAll, beforeAll, describe, expect, test as it } from "vitest";
+
+describe("S3 MD5 Fallback for DeleteObjects", () => {
+  let s3: S3;
+  let Bucket: string;
+  const testFiles = ["md5-test-1.txt", "md5-test-2.txt"];
+
+  beforeAll(async () => {
+    s3 = new S3({ region: "us-west-2" });
+    Bucket = `md5-fallback-test-${Date.now()}`;
+
+    try {
+      await s3.send(new CreateBucketCommand({ Bucket }));
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+
+      for (const Key of testFiles) {
+        await s3.send(
+          new PutObjectCommand({
+            Bucket,
+            Key,
+            Body: "test content",
+          })
+        );
+      }
+    } catch (err) {
+      console.error("Setup failed:", err);
+      throw err;
+    }
+  });
+
+  afterAll(async () => {
+    try {
+      await s3.send(
+        new DeleteObjectsCommand({
+          Bucket,
+          Delete: {
+            Objects: testFiles.map((Key) => ({ Key })),
+          },
+        })
+      );
+      await s3.send(new DeleteBucketCommand({ Bucket }));
+    } catch (error) {
+      console.error("Cleanup failed:", error);
+    }
+  });
+
+  it("should use CRC32 checksum by default for DeleteObjects", async () => {
+    const response = await s3.send(
+      new DeleteObjectsCommand({
+        Bucket,
+        Delete: {
+          Objects: [{ Key: testFiles[0] }],
+        },
+      })
+    );
+
+    // operation successfully deleted exactly one object (CRC32 being used)
+    expect(response.Deleted?.length).toBe(1);
+  });
+
+  it("should use MD5 checksum for DeleteObjects with middleware", async () => {
+    const md5S3Client = new S3({ region: "us-west-2" });
+    let md5Added = false;
+    let crc32Removed = false;
+
+    md5S3Client.middlewareStack.add(
+      (next) => async (args) => {
+        // Check if this is a DeleteObjects command
+        const isDeleteObjects = args.constructor?.name === "DeleteObjectsCommand" || args.input?.Delete !== undefined;
+
+        if (!isDeleteObjects) {
+          return next(args);
+        }
+
+        const result = await next(args);
+
+        const headers = args.request.headers;
+
+        // Remove checksum headers
+        Object.keys(headers).forEach((header) => {
+          if (
+            header.toLowerCase().startsWith("x-amz-checksum-") ||
+            header.toLowerCase().startsWith("x-amz-sdk-checksum-")
+          ) {
+            delete headers[header];
+            crc32Removed = true;
+          }
+        });
+
+        // Add MD5
+        if (args.request.body) {
+          const bodyContent = Buffer.from(args.request.body);
+          const md5Hash = createHash("md5").update(bodyContent).digest("base64");
+          headers["Content-MD5"] = md5Hash;
+          md5Added = true;
+        }
+
+        return result;
+      },
+      {
+        step: "finalizeRequest",
+        name: "addMD5Checksum",
+      }
+    );
+
+    const response = await md5S3Client.send(
+      new DeleteObjectsCommand({
+        Bucket,
+        Delete: {
+          Objects: [{ Key: testFiles[1] }],
+        },
+      })
+    );
+
+    // If MD5 wasn't properly set, this call will fail
+    expect(response.Deleted?.length).toBe(1);
+    expect(md5Added).toBe(true);
+    expect(crc32Removed).toBe(true);
+  });
+});

supplemental-docs/MD5_FALLBACK.md

@@ -30,8 +30,12 @@ export function createS3ClientWithMD5() {
         return next(args);
       }
 
-      // Remove any checksum headers
+      const result = await next(args);
+
+      // Modify the final request headers
       const headers = args.request.headers;
+
+      // Remove any checksum headers
       Object.keys(headers).forEach((header) => {
         if (
           header.toLowerCase().startsWith("x-amz-checksum-") ||
@@ -48,10 +52,10 @@ export function createS3ClientWithMD5() {
         headers["Content-MD5"] = md5Hash;
       }
 
-      return next(args);
+      return result;
     },
     {
-      step: "build",
+      step: "finalizeRequest", // Run after all other request modifications
       name: "addMD5Checksum",
     }
   );
@@ -92,14 +96,17 @@ try {
 The solution adds middleware to the S3 client that:
 
 1. Detects DeleteObjects operations
-2. Removes any checksum headers
-3. Calculates an MD5 hash of the request body (in the `build` step of the request lifecycle, as per the middleware implementation above)
-4. Adds the MD5 hash as a Content-MD5 header
+2. Lets the SDK add its default headers
+3. Removes any checksum headers in the finalizeRequest step
+4. Calculates an MD5 hash of the request body
+5. Adds the MD5 hash as a Content-MD5 header
+
+This sequence ensures that we properly replace the current checksums with the MD5 checksum.
 
 ## Usage Notes
 
 - The client can be configured with additional options as needed (region, credentials, etc.)
-- If your S3-compatible service supports the SDK's new checksum options or adds support in the future, you should use the standard S3 client instead
+- If your S3-compatible service supports the SDK's new checksum options or adds support in the future, you should use the standard S3 client instead.
 
 ## Debugging