Fix the uniqueItems implementation to accommodate non-primitive values

smithy-typescript-ssdk-libs/.prettierignore

@@ -0,0 +1 @@
+dist/

smithy-typescript-ssdk-libs/server-apigateway/package.json

@@ -11,8 +11,8 @@
     "postbuild": "rimraf dist/types/ts3.4 && downlevel-dts dist/types dist/types/ts3.4",
     "test": "jest --passWithNoTests",
     "clean": "rimraf dist",
-    "lint": "yarn run eslint -c ../.eslintrc.js **/src/**/*.ts",
-    "format": "prettier --config ../prettier.config.js --write **/*.{ts,js,md,json}"
+    "lint": "yarn run eslint -c ../.eslintrc.js \"src/**/*.ts\"",
+    "format": "prettier --config ../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\""
   },
   "repository": {
     "type": "git",

smithy-typescript-ssdk-libs/server-common/package.json

@@ -11,7 +11,8 @@
     "postbuild": "rimraf dist/types/ts3.4 && downlevel-dts dist/types dist/types/ts3.4",
     "test": "jest",
     "clean": "rimraf dist",
-    "format": "prettier --config ../prettier.config.js --write **/*.{ts,js,md,json}"
+    "lint": "yarn run eslint -c ../.eslintrc.js \"src/**/*.ts\"",
+    "format": "prettier --config ../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\""
   },
   "repository": {
     "type": "git",

smithy-typescript-ssdk-libs/server-common/src/index.ts

@@ -17,6 +17,7 @@ export * as httpbinding from "./httpbinding";
 export * from "./accept";
 export * from "./errors";
 export * from "./validation";
+export * from "./unique";
 
 import { HttpRequest, HttpResponse } from "@aws-sdk/protocol-http";
 import { SmithyException } from "@aws-sdk/smithy-client";

smithy-typescript-ssdk-libs/server-common/src/unique.spec.ts

@@ -0,0 +1,141 @@
+/*
+ *  Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License").
+ *  You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *   http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+import * as util from "util";
+
+import { findDuplicates, Input } from "./unique";
+
+describe("findDuplicates", () => {
+  describe("finds duplicates in", () => {
+    it("strings", () => {
+      expect(findDuplicates(["a", "b", "c", "a", "b", "a"])).toEqual(["a", "b"]);
+      expect(findDuplicates(["x", "y", "z", "a", "b", "c", "a", "b"])).toEqual(["a", "b"]);
+    });
+    it("numbers", () => {
+      expect(findDuplicates([1, 2, 3, 4, 1, 2])).toEqual([1, 2]);
+    });
+    it("booleans", () => {
+      expect(findDuplicates([true, false, true])).toEqual([true]);
+    });
+    it("arrays", () => {
+      expect(
+        findDuplicates([
+          [5, 6],
+          [2, 3],
+          [1, 2],
+          [3, 4],
+          [1, 2],
+        ])
+      ).toEqual([[1, 2]]);
+    });
+    it("Dates", () => {
+      expect(findDuplicates([new Date(1000), new Date(2000), new Date(1000)])).toEqual([new Date(1000)]);
+    });
+    it("Blobs", () => {
+      expect(findDuplicates([Uint8Array.of(1, 2, 3), Uint8Array.of(4, 5, 6), Uint8Array.of(4, 5, 6)])).toEqual([
+        Uint8Array.of(4, 5, 6),
+      ]);
+    });
+    it("objects", () => {
+      expect(findDuplicates([{ a: "b" }, { b: [1, 2, 3] }, { a: "b" }, { a: "b" }])).toEqual([{ a: "b" }]);
+      expect(findDuplicates([{ a: "b" }, { b: 1, c: 2 }, { c: 2, b: 1 }])).toEqual([{ b: 1, c: 2 }]);
+    });
+    it("deeply nested objects", () => {
+      expect(
+        findDuplicates([
+          { a: { b: { c: [1, { d: 2 }, [3]] } } },
+          2,
+          [3, 4],
+          { b: "c" },
+          { a: { b: { c: [1, { d: 2 }, [3]] } } },
+        ])
+      ).toEqual([{ a: { b: { c: [1, { d: 2 }, [3]] } } }]);
+    });
+  });
+  describe("correctly does not find duplicates in", () => {
+    it("strings", () => {
+      expect(findDuplicates(["a", "b", "c"])).toEqual([]);
+    });
+    it("numbers", () => {
+      expect(findDuplicates([1, 2, 3, 4])).toEqual([]);
+      expect(findDuplicates([1, 2, "1", "2"])).toEqual([]);
+    });
+    it("booleans", () => {
+      expect(findDuplicates([true, false])).toEqual([]);
+      expect(findDuplicates([true, false, "true", "false"])).toEqual([]);
+    });
+    it("arrays", () => {
+      expect(
+        findDuplicates([
+          [1, 2],
+          [2, 3],
+          [3, 4],
+        ])
+      ).toEqual([]);
+      expect(
+        findDuplicates([
+          [1, 2],
+          ["1", "2"],
+        ])
+      ).toEqual([]);
+    });
+    it("objects", () => {
+      expect(findDuplicates([{ a: "b" }, { b: [1, 2, 3] }])).toEqual([]);
+      expect(findDuplicates([{ a: 1 }, { a: "1" }])).toEqual([]);
+    });
+    it("Dates", () => {
+      expect(findDuplicates([new Date(100), new Date(200), new Date(101)])).toEqual([]);
+    });
+    it("blobs", () => {
+      expect(findDuplicates([Uint8Array.of(1, 2, 3), Uint8Array.of(1, 2)])).toEqual([]);
+    });
+  });
+
+  // This is relatively slow and may be flaky if the input size is tuned to let it run reasonably fast
+  it.skip("is faster than the naive implementation", () => {
+    const input: Input[] = [true, false, 1, 2, 3, 4, 5, 6];
+    for (let i = 0; i < 10_000; i++) {
+      input.push({ a: [1, 2, 3, i], b: { nested: [true] } });
+    }
+
+    const uniqueStart = Date.now();
+    expect(findDuplicates(input)).toEqual([]);
+    const uniqueEnd = Date.now();
+    const uniqueDuration = (uniqueEnd - uniqueStart) / 1000;
+
+    console.log(`findDuplicates finished in ${uniqueDuration} seconds`);
+
+    const naiveStart = Date.now();
+    expect(naivefindDuplicates(input)).toEqual([]);
+    const naiveEnd = Date.now();
+    const naiveDuration = (naiveEnd - naiveStart) / 1000;
+
+    console.log(`naivefindDuplicates finished in ${naiveDuration} seconds`);
+
+    expect(naiveDuration).toBeGreaterThan(uniqueDuration);
+  });
+
+  // This isn't even correct! It should be even slower, it just returns the first duplicate!
+  function naivefindDuplicates(input: Array<Input>): Input | undefined {
+    for (let i = 0; i < input.length - 1; i++) {
+      for (let j = i + 1; j < input.length; j++) {
+        if (util.isDeepStrictEqual(input[i], input[j])) {
+          return [input[i]];
+        }
+      }
+    }
+    return [];
+  }
+});

smithy-typescript-ssdk-libs/server-common/src/unique.ts

@@ -0,0 +1,101 @@
+/*
+ *  Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License").
+ *  You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *   http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+import { createHash } from "crypto";
+import * as util from "util";
+
+/**
+ * A shortcut for JSON and Smithy primitives, as well as documents and Smithy-
+ * modeled structures composed of those primitives
+ */
+export type Input = { [key: string]: Input } | Array<Input> | Date | Uint8Array | string | number | boolean | null;
+
+/**
+ * Returns an array of duplicated values in the input. This is equivalent to using
+ * {@link util#isDeepStrictEqual} to compare every member of the input to all the
+ * other members, but with an optimization to make the runtime complexity O(n)
+ * instead of O(n^2).
+ *
+ * @param input an array of {@link Input}
+ * @return an array containing one instance of every duplicated member of the input,
+ *         or an empty array if there are no duplicates
+ */
+export const findDuplicates = (input: Array<Input>): Array<Input> => {
+  const potentialCollisions: { [hash: string]: { value: Input; alreadyFound: boolean }[] } = {};
+  const collisions: Array<Input> = [];
+
+  for (const value of input) {
+    const valueHash = hash(value);
+    if (!potentialCollisions.hasOwnProperty(valueHash)) {
+      potentialCollisions[valueHash] = [{ value: value, alreadyFound: false }];
+    } else {
+      let duplicateFound = false;
+      for (const potentialCollision of potentialCollisions[valueHash]) {
+        if (util.isDeepStrictEqual(value, potentialCollision.value)) {
+          duplicateFound = true;
+          if (!potentialCollision.alreadyFound) {
+            collisions.push(value);
+            potentialCollision.alreadyFound = true;
+          }
+        }
+      }
+      if (!duplicateFound) {
+        potentialCollisions[valueHash].push({ value: value, alreadyFound: false });
+      }
+    }
+  }
+  return collisions;
+};
+
+const hash = (input: Input): string => {
+  return createHash("sha256").update(canonicalize(input)).digest("base64");
+};
+
+/**
+ * Since node's hash functions operate on strings or buffers, we need a canonical format for
+ * our objects in order to hash them correctly. This function turns them into string representations
+ * where the types are encoded in order to avoid ambiguity, for instance, between the string "1" and
+ * the number 1. This method sorts object keys lexicographically in order to maintain consistency.
+ *
+ * This doesn't just call JSON.stringify because we want to have firm control over the ordering of map
+ * keys and the handling of blobs and dates
+ *
+ * @param input a JSON-like object
+ * @return a canonical string representation
+ */
+const canonicalize = (input: Input): string => {
+  if (input === null) {
+    return "null()";
+  }
+  if (typeof input === "string" || typeof input === "number" || typeof input === "boolean") {
+    return `${typeof input}(${input.toString()})`;
+  }
+  if (Array.isArray(input)) {
+    return "array(" + input.map((i) => canonicalize(i)).join(",") + ")";
+  }
+  if (input instanceof Date) {
+    return "date(" + input.getTime() + ")";
+  }
+  if (input instanceof Uint8Array) {
+    // hashing the blob just to avoid allocating another base64 copy of its data
+    return "blob(" + createHash("sha256").update(input).digest("base64") + ")";
+  }
+
+  const contents: Array<string> = [];
+  for (const key of Object.keys(input).slice().sort()) {
+    contents.push("key(" + key + ")->value(" + canonicalize(input[key]) + ")");
+  }
+  return "map(" + contents.join(",") + ")";
+};

smithy-typescript-ssdk-libs/server-common/src/validation/validators.spec.ts

@@ -231,4 +231,12 @@ describe("uniqueItems", () => {
       path: "aField",
     });
   });
+  describe("supports objects", () => {
+    expect(validator.validate([{ a: 1 }, { b: 2 }], "aField")).toBeNull();
+    expect(validator.validate([{ a: 1 }, { b: 2 }, { a: 1 }], "aField")).toEqual({
+      constraintType: "uniqueItems",
+      failureValue: [{ a: 1 }],
+      path: "aField",
+    });
+  });
 });

smithy-typescript-ssdk-libs/server-common/src/validation/validators.ts

@@ -15,6 +15,7 @@
 
 import { RE2 } from "re2-wasm";
 
+import { findDuplicates } from "../unique";
 import {
   EnumValidationFailure,
   LengthValidationFailure,
@@ -300,17 +301,9 @@ export class UniqueItemsValidator implements SingleConstraintValidator<Array<any
       return null;
     }
 
-    const repeats = new Set<any>();
-    const uniqueValues = new Set<any>();
-    for (const i of input) {
-      if (uniqueValues.has(i)) {
-        repeats.add(i);
-      } else {
-        uniqueValues.add(i);
-      }
-    }
+    const repeats = findDuplicates(input);
 
-    if (repeats.size > 0) {
+    if (repeats.length > 0) {
       return {
         constraintType: "uniqueItems",
         path: path,