Fix the uniqueItems implementation to accommodate non-primitive values

JavaScript Sets do not work in the same fashion as HashSets do in Java, in that
object equivalence is determined by reference, not by value. In order to work
around this, this commit adds a duplicate finder that works on the same
principals as a value-based hashtable: it creates a hash of the object's value
to bucket objects and then uses deep equality to determine if a hash collision
is actually a duplicate value.

Author: adamthom-amzn

smithy-typescript-ssdk-libs/.prettierignore

@@ -0,0 +1 @@
+dist/

smithy-typescript-ssdk-libs/server-apigateway/package.json

@@ -11,8 +11,8 @@
     "postbuild": "rimraf dist/types/ts3.4 && downlevel-dts dist/types dist/types/ts3.4",
     "test": "jest --passWithNoTests",
     "clean": "rimraf dist",
-    "lint": "yarn run eslint -c ../.eslintrc.js **/src/**/*.ts",
-    "format": "prettier --config ../prettier.config.js --write **/*.{ts,js,md,json}"
+    "lint": "yarn run eslint -c ../.eslintrc.js \"src/**/*.ts\"",
+    "format": "prettier --config ../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\""
   },
   "repository": {
     "type": "git",

smithy-typescript-ssdk-libs/server-common/package.json

@@ -11,7 +11,8 @@
     "postbuild": "rimraf dist/types/ts3.4 && downlevel-dts dist/types dist/types/ts3.4",
     "test": "jest",
     "clean": "rimraf dist",
-    "format": "prettier --config ../prettier.config.js --write **/*.{ts,js,md,json}"
+    "lint": "yarn run eslint -c ../.eslintrc.js \"src/**/*.ts\"",
+    "format": "prettier --config ../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\""
   },
   "repository": {
     "type": "git",

smithy-typescript-ssdk-libs/server-common/src/index.ts

@@ -17,6 +17,7 @@ export * as httpbinding from "./httpbinding";
 export * from "./accept";
 export * from "./errors";
 export * from "./validation";
+export * from "./unique";
 
 import { HttpRequest, HttpResponse } from "@aws-sdk/protocol-http";
 import { SmithyException } from "@aws-sdk/smithy-client";

smithy-typescript-ssdk-libs/server-common/src/unique.spec.ts

@@ -0,0 +1,141 @@
+/*
+ *  Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License").
+ *  You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *   http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+import * as util from "util";
+
+import { findDuplicates, Input } from "./unique";
+
+describe("findDuplicates", () => {
+  describe("finds duplicates in", () => {
+    it("strings", () => {
+      expect(findDuplicates(["a", "b", "c", "a", "b", "a"])).toEqual(["a", "b"]);
+      expect(findDuplicates(["x", "y", "z", "a", "b", "c", "a", "b"])).toEqual(["a", "b"]);
+    });
+    it("numbers", () => {
+      expect(findDuplicates([1, 2, 3, 4, 1, 2])).toEqual([1, 2]);
+    });
+    it("booleans", () => {
+      expect(findDuplicates([true, false, true])).toEqual([true]);
+    });
+    it("arrays", () => {
+      expect(
+        findDuplicates([
+          [5, 6],
+          [2, 3],
+          [1, 2],
+          [3, 4],
+          [1, 2],
+        ])
+      ).toEqual([[1, 2]]);
+    });
+    it("Dates", () => {
+      expect(findDuplicates([new Date(1000), new Date(2000), new Date(1000)])).toEqual([new Date(1000)]);
+    });
+    it("Blobs", () => {
+      expect(findDuplicates([Uint8Array.of(1, 2, 3), Uint8Array.of(4, 5, 6), Uint8Array.of(4, 5, 6)])).toEqual([
+        Uint8Array.of(4, 5, 6),
+      ]);
+    });
+    it("objects", () => {
+      expect(findDuplicates([{ a: "b" }, { b: [1, 2, 3] }, { a: "b" }, { a: "b" }])).toEqual([{ a: "b" }]);
+      expect(findDuplicates([{ a: "b" }, { b: 1, c: 2 }, { c: 2, b: 1 }])).toEqual([{ b: 1, c: 2 }]);
+    });
+    it("deeply nested objects", () => {
+      expect(
+        findDuplicates([
+          { a: { b: { c: [1, { d: 2 }, [3]] } } },
+          2,
+          [3, 4],
+          { b: "c" },
+          { a: { b: { c: [1, { d: 2 }, [3]] } } },
+        ])
+      ).toEqual([{ a: { b: { c: [1, { d: 2 }, [3]] } } }]);
+    });
+  });
+  describe("correctly does not find duplicates in", () => {
+    it("strings", () => {
+      expect(findDuplicates(["a", "b", "c"])).toEqual([]);
+    });
+    it("numbers", () => {
+      expect(findDuplicates([1, 2, 3, 4])).toEqual([]);
+      expect(findDuplicates([1, 2, "1", "2"])).toEqual([]);
+    });
+    it("booleans", () => {
+      expect(findDuplicates([true, false])).toEqual([]);
+      expect(findDuplicates([true, false, "true", "false"])).toEqual([]);
+    });
+    it("arrays", () => {
+      expect(
+        findDuplicates([
+          [1, 2],
+          [2, 3],
+          [3, 4],
+        ])
+      ).toEqual([]);
+      expect(
+        findDuplicates([
+          [1, 2],
+          ["1", "2"],
+        ])
+      ).toEqual([]);
+    });
+    it("objects", () => {
+      expect(findDuplicates([{ a: "b" }, { b: [1, 2, 3] }])).toEqual([]);
+      expect(findDuplicates([{ a: 1 }, { a: "1" }])).toEqual([]);
+    });
+    it("Dates", () => {
+      expect(findDuplicates([new Date(100), new Date(200), new Date(101)])).toEqual([]);
+    });
+    it("blobs", () => {
+      expect(findDuplicates([Uint8Array.of(1, 2, 3), Uint8Array.of(1, 2)])).toEqual([]);
+    });
+  });
+
+  // This is relatively slow and may be flaky if the input size is tuned to let it run reasonably fast
+  it.skip("is faster than the naive implementation", () => {
+    const input: Input[] = [true, false, 1, 2, 3, 4, 5, 6];
+    for (let i = 0; i < 10_000; i++) {
+      input.push({ a: [1, 2, 3, i], b: { nested: [true] } });
+    }
+
+    const uniqueStart = Date.now();
+    expect(findDuplicates(input)).toEqual([]);
+    const uniqueEnd = Date.now();
+    const uniqueDuration = (uniqueEnd - uniqueStart) / 1000;
+
+    console.log(`findDuplicates finished in ${uniqueDuration} seconds`);
+
+    const naiveStart = Date.now();
+    expect(naivefindDuplicates(input)).toEqual([]);
+    const naiveEnd = Date.now();
+    const naiveDuration = (naiveEnd - naiveStart) / 1000;
+
+    console.log(`naivefindDuplicates finished in ${naiveDuration} seconds`);
+
+    expect(naiveDuration).toBeGreaterThan(uniqueDuration);
+  });
+
+  // This isn't even correct! It should be even slower, it just returns the first duplicate!
+  function naivefindDuplicates(input: Array<Input>): Input | undefined {
+    for (let i = 0; i < input.length - 1; i++) {
+      for (let j = i + 1; j < input.length; j++) {
+        if (util.isDeepStrictEqual(input[i], input[j])) {
+          return [input[i]];
+        }
+      }
+    }
+    return [];
+  }
+});

smithy-typescript-ssdk-libs/server-common/src/unique.ts

@@ -0,0 +1,96 @@
+/*
+ *  Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License").
+ *  You may not use this file except in compliance with the License.
+ *  A copy of the License is located at
+ *
+ *   http://aws.amazon.com/apache2.0
+ *
+ *  or in the "license" file accompanying this file. This file is distributed
+ *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *  express or implied. See the License for the specific language governing
+ *  permissions and limitations under the License.
+ */
+
+import { createHash } from "crypto";
+import * as util from "util";
+
+/**
+ * A shortcut for document-like objects or Smithy structures.
+ */
+export type Input = { [key: string]: Input } | Array<Input> | Date | Uint8Array | string | number | boolean | null;
+
+/**
+ * Returns an array of duplicated values in the input. Unlike equals, duplicate
+ * is determined by semantic equivalence (two objects representing the same
+ * data, not just two objects at the same location in memory).
+ *
+ * @param input an array of JSON- or Smithy-modeled objects.
+ */
+export const findDuplicates = (input: Array<Input>): Array<Input> => {
+  const potentialCollisions: { [hash: string]: { value: Input; alreadyFound: boolean }[] } = {};
+  const collisions: Array<Input> = [];
+
+  for (const value of input) {
+    const valueHash = hash(value);
+    if (!potentialCollisions.hasOwnProperty(valueHash)) {
+      potentialCollisions[valueHash] = [{ value: value, alreadyFound: false }];
+    } else {
+      let duplicateFound = false;
+      for (const potentialCollision of potentialCollisions[valueHash]) {
+        if (util.isDeepStrictEqual(value, potentialCollision.value)) {
+          duplicateFound = true;
+          if (!potentialCollision.alreadyFound) {
+            collisions.push(value);
+          }
+          potentialCollision.alreadyFound = true;
+        }
+      }
+      if (!duplicateFound) {
+        potentialCollisions[valueHash].push({ value: value, alreadyFound: false });
+      }
+    }
+  }
+  return collisions;
+};
+
+const hash = (input: Input): string => {
+  return createHash("sha256").update(canonicalize(input)).digest("base64");
+};
+
+/**
+ * Since node's hash functions operate on strings or buffers, we need a canonical format for
+ * our objects in order to hash them correctly. This function turns them into string representations
+ * where the types are encoded in order to avoid ambiguity, for instance, between the string "1" and
+ * the number 1. This method sorts object keys lexicographically in order to maintain consistency.
+ *
+ * This doesn't just call JSON.stringify because we want to have firm control over the ordering of map
+ * keys and the handling of blobs
+ *
+ * @param input a JSON-like object
+ * @return a canonical string representation
+ */
+const canonicalize = (input: Input): string => {
+  if (input === null) {
+    return "null()";
+  }
+  if (typeof input === "string" || typeof input === "number" || typeof input === "boolean") {
+    return `${typeof input}(${input.toString()})`;
+  }
+  if (Array.isArray(input)) {
+    return "array(" + input.map((i) => canonicalize(i)).join(",") + ")";
+  }
+  if (input instanceof Date) {
+    return "date(" + input.getTime() + ")";
+  }
+  if (input instanceof Uint8Array) {
+    return "blob(" + createHash("sha256").update(input).digest("base64") + ")";
+  }
+
+  const contents: Array<string> = [];
+  for (const key of Object.keys(input).slice().sort()) {
+    contents.push("key(" + key + ")->value(" + canonicalize(input[key]) + ")");
+  }
+  return "map(" + contents.join(",") + ")";
+};

smithy-typescript-ssdk-libs/server-common/src/validation/validators.spec.ts

@@ -231,4 +231,12 @@ describe("uniqueItems", () => {
       path: "aField",
     });
   });
+  describe("supports documents", () => {
+    expect(validator.validate([{ a: 1 }, { b: 2 }], "aField")).toBeNull();
+    expect(validator.validate([{ a: 1 }, { b: 2 }, { a: 1 }], "aField")).toEqual({
+      constraintType: "uniqueItems",
+      failureValue: [{ a: 1 }],
+      path: "aField",
+    });
+  });
 });

smithy-typescript-ssdk-libs/server-common/src/validation/validators.ts

@@ -15,6 +15,7 @@
 
 import { RE2 } from "re2-wasm";
 
+import { findDuplicates } from "../unique";
 import {
   EnumValidationFailure,
   LengthValidationFailure,
@@ -300,17 +301,9 @@ export class UniqueItemsValidator implements SingleConstraintValidator<Array<any
       return null;
     }
 
-    const repeats = new Set<any>();
-    const uniqueValues = new Set<any>();
-    for (const i of input) {
-      if (uniqueValues.has(i)) {
-        repeats.add(i);
-      } else {
-        uniqueValues.add(i);
-      }
-    }
+    const repeats = findDuplicates(input);
 
-    if (repeats.size > 0) {
+    if (repeats.length > 0) {
       return {
         constraintType: "uniqueItems",
         path: path,