feat(endpoint): update existing packages for endpoints v2

Author: kuhe

clients/client-s3/src/commands/GetObjectCommand.ts

@@ -257,8 +257,6 @@ export class GetObjectCommand extends $Command<GetObjectCommandInput, GetObjectC
 
     const stack = clientStack.concat(this.middlewareStack);
 
-    console.log(stack.identify());
-
     const { logger } = configuration;
     const clientName = "S3Client";
     const commandName = "GetObjectCommand";

lib/lib-storage/package.json

@@ -24,6 +24,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@aws-sdk/middleware-endpoint": "*",
     "@aws-sdk/smithy-client": "*",
     "buffer": "5.6.0",
     "events": "3.3.0",
@@ -37,6 +38,7 @@
   "devDependencies": {
     "@aws-sdk/abort-controller": "*",
     "@aws-sdk/client-s3": "*",
+    "@aws-sdk/types": "*",
     "@tsconfig/recommended": "1.0.1",
     "@types/node": "^14.11.2",
     "concurrently": "7.0.0",

lib/lib-storage/src/Upload.ts

@@ -13,8 +13,10 @@ import {
   Tag,
   UploadPartCommand,
 } from "@aws-sdk/client-s3";
+import { EndpointResolvedConfig, getEndpointFromInstructions, toEndpointV1 } from "@aws-sdk/middleware-endpoint";
 import { HttpRequest } from "@aws-sdk/protocol-http";
 import { extendedEncodeURIComponent } from "@aws-sdk/smithy-client";
+import { Endpoint } from "@aws-sdk/types";
 import { EventEmitter } from "events";
 
 import { byteLength } from "./bytelength";
@@ -101,7 +103,8 @@ export class Upload extends EventEmitter {
     this.isMultiPart = false;
     const params = { ...this.params, Body: dataPart.data };
 
-    const requestHandler = this.client.config.requestHandler;
+    const clientConfig = this.client.config;
+    const requestHandler = clientConfig.requestHandler;
     const eventEmitter: EventEmitter | null = requestHandler instanceof EventEmitter ? requestHandler : null;
     const uploadEventListener = (event: ProgressEvent) => {
       this.bytesUploadedSoFar = event.loaded;
@@ -120,14 +123,22 @@ export class Upload extends EventEmitter {
       eventEmitter.on("xhr.upload.progress", uploadEventListener);
     }
 
-    const [putResult, endpoint] = await Promise.all([
-      this.client.send(new PutObjectCommand(params)),
-      this.client.config?.endpoint?.(),
-    ]);
+    const resolved = await Promise.all([this.client.send(new PutObjectCommand(params)), clientConfig?.endpoint?.()]);
+    const putResult = resolved[0];
+    let endpoint: Endpoint = resolved[1];
 
     if (!endpoint) {
-      // TODO(endpointsv2): handle endpoint v2
-      throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()".');
+      endpoint = toEndpointV1(
+        await getEndpointFromInstructions(
+          params,
+          PutObjectCommand,
+          clientConfig
+        )
+      );
+    }
+
+    if (!endpoint) {
+      throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()" nor EndpointsV2.');
     }
 
     if (eventEmitter !== null) {

packages/middleware-endpoint/package.json

@@ -5,6 +5,7 @@
     "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:es": "tsc -p tsconfig.es.json",
+    "build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
     "build:types": "tsc -p tsconfig.types.json",
     "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
     "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
@@ -19,6 +20,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@aws-sdk/protocol-http": "*",
     "@aws-sdk/signature-v4": "*",
     "@aws-sdk/types": "*",
     "@aws-sdk/util-config-provider": "*",

packages/middleware-endpoint/src/adaptors/getEndpointFromConfig.ts

@@ -0,0 +1,65 @@
+import { EndpointParameters, EndpointV2, HandlerExecutionContext } from "@aws-sdk/types";
+
+import { EndpointResolvedConfig } from "../resolveEndpointConfig";
+import { EndpointParameterInstructions } from "../types";
+
+/**
+ * 
+ * This step in the endpoint resolution process is exposed as a function
+ * to allow certain packages such as s3 signer, lib-upload to get
+ * the V2 Endpoint associated to an instance of some (operation) command.
+ * 
+ * @private
+ * 
+ */
+export const getEndpointFromInstructions = async (
+  commandInput: any,
+  instructionsSupplier: Partial<{
+    instructions(): EndpointParameterInstructions
+  }>, 
+  clientConfig: Partial<EndpointResolvedConfig>,
+  context?: HandlerExecutionContext
+): Promise<EndpointV2> => {
+  const endpointParams: EndpointParameters = {};
+  const instructions: EndpointParameterInstructions = instructionsSupplier?.instructions() || {};
+
+  if (typeof clientConfig.endpointProvider !== "function") {
+    throw new Error("config.endpointProvider is not set.");
+  }
+
+  for (const [name, instruction] of Object.entries(instructions)) {
+    switch (instruction.type) {
+      case "staticContextParams":
+        endpointParams[name] = instruction.value;
+        break;
+      case "contextParams":
+        endpointParams[name] = commandInput[instruction.name];
+        break;
+      case "clientContextParams":
+      case "builtInParams":
+        endpointParams[name] = await createConfigProvider(instruction.name, clientConfig)();
+        break;
+      default:
+        throw new Error("Unrecognized endpoint parameter instruction: " + JSON.stringify(instruction));
+    }
+  }
+
+  const endpoint: EndpointV2 = clientConfig.endpointProvider!(endpointParams, context);
+  return endpoint;
+};
+
+/**
+ * @private
+ */
+ const createConfigProvider = (configKey: string, config: EndpointResolvedConfig & any) => {
+  return async function configProvider() {
+    if (!(configKey in config)) {
+      throw new Error(`The config key ${configKey} was not found in the config object.`);
+    }
+    const configValue = config[configKey];
+    if (typeof configValue === "function") {
+      return configValue();
+    }
+    return configValue;
+  };
+};

packages/middleware-endpoint/src/adaptors/toEndpointV1.ts

@@ -0,0 +1,14 @@
+import { Endpoint, EndpointV2 } from "@aws-sdk/types";
+import { parseUrl } from "@aws-sdk/url-parser";
+
+export const toEndpointV1 = (endpoint: string | Endpoint | EndpointV2): Endpoint => {
+  if (typeof endpoint === "object") {
+    if ("url" in endpoint) {
+      // v2
+      return parseUrl(endpoint.url);
+    }
+    // v1
+    return endpoint;
+  }
+  return parseUrl(endpoint);
+};

packages/middleware-endpoint/src/endpointMiddleware.ts

@@ -1,24 +1,54 @@
+import { HttpRequest } from "@aws-sdk/protocol-http";
+import { parseQueryString } from "@aws-sdk/querystring-parser";
 import {
+  EndpointV2,
+  HandlerExecutionContext,
   MetadataBearer,
   SerializeHandler,
   SerializeHandlerArguments,
   SerializeHandlerOutput,
   SerializeMiddleware,
 } from "@aws-sdk/types";
 
-import { EndpointParameterInstruction } from "./types";
+import { getEndpointFromInstructions } from "./adaptors/getEndpointFromConfig";
+import { EndpointResolvedConfig } from "./resolveEndpointConfig";
+import { EndpointParameterInstructions } from "./types";
 
-export const endpointMiddleware = (options: {
-  config: any; // TODO(endpointsV2): should be ResolvedEndpointConfig interface
-  instruction: EndpointParameterInstruction;
+/**
+ * @private
+ */
+export const endpointMiddleware = ({
+  config,
+  instructions,
+}: {
+  config: EndpointResolvedConfig;
+  instructions: EndpointParameterInstructions;
 }): SerializeMiddleware<any, any> => {
-  return <Output extends MetadataBearer>(next: SerializeHandler<any, Output>): SerializeHandler<any, Output> =>
+  return <Output extends MetadataBearer>(
+      next: SerializeHandler<any, Output>,
+      context: HandlerExecutionContext
+    ): SerializeHandler<any, Output> =>
     async (args: SerializeHandlerArguments<any>): Promise<SerializeHandlerOutput<Output>> => {
-      // TODO(endpointsV2): resolve async providers from client config and static values according to
-      // instruction from input to populate endpoint parameters
+      if (HttpRequest.isInstance(args.request)) {
+        const { request } = args;
+
+        const endpoint: EndpointV2 = await getEndpointFromInstructions(args.input, instructions, config, context);
+
+        context.endpointV2 = endpoint;
+        context.authSchemes = endpoint.properties?.authSchemes;
+
+        request.headers = Object.entries(endpoint.headers || {}).reduce((headers, [name, values]) => {
+          headers[name] = values.join(",");
+          return headers;
+        }, {} as Record<string, string>);
+        request.hostname = endpoint.url.hostname;
+        request.path = endpoint.url.pathname;
+        request.port = parseInt(endpoint.url.port);
+        request.protocol = endpoint.url.protocol;
+        request.query = parseQueryString(endpoint.url.search);
+      }
       return next({
         ...args,
       });
     };
 };
-

packages/middleware-endpoint/src/getEndpointPlugin.ts

@@ -1,24 +1,25 @@
 import { Pluggable, SerializeHandlerOptions } from "@aws-sdk/types";
 
 import { endpointMiddleware } from "./endpointMiddleware";
-import { EndpointParameterInstruction } from "./types";
+import { EndpointResolvedConfig } from "./resolveEndpointConfig";
+import { EndpointParameterInstructions } from "./types";
 
 export const endpointMiddlewareOptions: SerializeHandlerOptions = {
   step: "serialize",
   tags: ["ENDPOINT_PARAMETERS", "ENDPOINT_V2", "ENDPOINT"],
-  name: "endpointMiddleware",
+  name: "endpointsV2Middleware",
   override: true,
 };
 
 export const getEndpointPlugin = (
-  config: any, //TODO(endpointsV2): should be ResolvedEndpointConfig interface
-  instruction: EndpointParameterInstruction
+  config: EndpointResolvedConfig,
+  instructions: EndpointParameterInstructions
 ): Pluggable<any, any> => ({
   applyToStack: (clientStack) => {
     clientStack.add(
       endpointMiddleware({
         config,
-        instruction,
+        instructions,
       }),
       endpointMiddlewareOptions
     );

packages/middleware-endpoint/src/index.ts

@@ -1,3 +1,5 @@
+export * from "./adaptors/getEndpointFromConfig";
+export * from "./adaptors/toEndpointV1";
 export * from "./endpointMiddleware";
 export * from "./getEndpointPlugin";
 export * from "./resolveEndpointConfig";

packages/middleware-endpoint/src/resolveEndpointConfig.ts

@@ -1,6 +1,8 @@
 import { Endpoint, EndpointParameters, EndpointV2, Logger, Provider, UrlParser } from "@aws-sdk/types";
 import { normalizeProvider } from "@aws-sdk/util-middleware";
 
+import { toEndpointV1 } from "./adaptors/toEndpointV1";
+
 /**
  * Endpoint config interfaces and resolver for Endpoints v2. They live in separate package to allow per-service onboarding.
  * When all services onboard the Endpoints v2, the resolver in config-resolver package can be removed.
@@ -102,5 +104,3 @@ export const resolveEndpointConfig = <T, P extends EndpointParameters = Endpoint
     useFipsEndpoint: normalizeProvider(input.useFipsEndpoint ?? false),
   };
 };
-
-declare const toEndpointV1: (endpoint: string | Endpoint | EndpointV2) => Endpoint; // TODO(endpointsV2) implementation

packages/middleware-endpoint/src/types.ts

@@ -1,4 +1,4 @@
-export interface EndpointParameterInstruction {
+export interface EndpointParameterInstructions {
   [name: string]:
     | BuiltInParamInstruction
     | ClientContextParamInstruction

packages/middleware-signing/src/configuration.spec.ts

@@ -22,8 +22,7 @@ describe("AuthConfig", () => {
       jest.clearAllMocks();
     });
 
-    // TODO(endpointsv2)
-    it.skip("should memoize custom credential provider", async () => {
+    it("should memoize custom credential provider", async () => {
       const { signer: signerProvider } = resolveAwsAuthConfig(inputParams);
       const signer = await signerProvider({ name: "sigv4", properties: {} });
       const request = new HttpRequest({});
@@ -34,8 +33,7 @@ describe("AuthConfig", () => {
       expect(inputParams.credentials).toBeCalledTimes(1);
     });
 
-    // TODO(endpointsv2)
-    it.skip("should refresh custom credential provider if expired", async () => {
+    it("should refresh custom credential provider if expired", async () => {
       const FOUR_MINUTES_AND_59_SEC = 299 * 1000;
       const input = {
         ...inputParams,
@@ -75,8 +73,7 @@ describe("AuthConfig", () => {
       jest.clearAllMocks();
     });
 
-    // TODO(endpointsv2)
-    it.skip("should memoize custom credential provider", async () => {
+    it("should memoize custom credential provider", async () => {
       const { signer: signerProvider } = resolveSigV4AuthConfig(inputParams);
       const signer = await signerProvider({ name: "sigv4", properties: {} });
       const request = new HttpRequest({});
@@ -87,8 +84,7 @@ describe("AuthConfig", () => {
       expect(inputParams.credentials).toBeCalledTimes(1);
     });
 
-    // TODO(endpointsv2)
-    it.skip("should refresh custom credential provider if expired", async () => {
+    it("should refresh custom credential provider if expired", async () => {
       const FOUR_MINUTES_AND_59_SEC = 299 * 1000;
       const input = {
         ...inputParams,