feat(endpoint): endpoints 2.0 existing package changes

codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java

@@ -72,8 +72,8 @@ public final class AddS3Config implements TypeScriptIntegration {
     );
 
     private static final String CRT_NOTIFICATION = "<p>Note: To supply the Multi-region Access Point (MRAP) to Bucket,"
-            + " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
-            + "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
+        + " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
+        + "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
 
     @Override
     public Model preprocessModel(Model model, TypeScriptSettings settings) {
@@ -118,17 +118,19 @@ public void addConfigInterfaceFields(
             return;
         }
         writer.writeDocs("Whether to escape request path when signing the request.")
-                .write("signingEscapePath?: boolean;\n");
+            .write("signingEscapePath?: boolean;\n");
         writer.writeDocs(
                 "Whether to override the request region with the region inferred from requested resource's ARN."
-                        + " Defaults to false.")
-                .addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
-                .write("useArnRegion?: boolean | Provider<boolean>;");
+                    + " Defaults to false.")
+            .addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
+            .write("useArnRegion?: boolean | Provider<boolean>;");
     }
 
     @Override
-    public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScriptSettings settings, Model model,
-            SymbolProvider symbolProvider, LanguageTarget target) {
+    public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(
+        TypeScriptSettings settings, Model model,
+        SymbolProvider symbolProvider, LanguageTarget target
+    ) {
         if (!isS3(settings.getService(model))) {
             return Collections.emptyMap();
         }
@@ -140,18 +142,18 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
                     writer.write("false");
                 }, "signerConstructor", writer -> {
                     writer.addDependency(AwsDependency.SIGNATURE_V4_MULTIREGION)
-                            .addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
-                                    AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
-                            .write("SignatureV4MultiRegion");
+                        .addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
+                            AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
+                        .write("SignatureV4MultiRegion");
                 });
             case NODE:
                 return MapUtils.of("useArnRegion", writer -> {
                     writer.addDependency(AwsDependency.NODE_CONFIG_PROVIDER)
-                            .addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
-                            .addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
-                            .addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
-                                    AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
-                            .write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
+                        .addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
+                        .addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
+                        .addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
+                            AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
+                        .write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
                 });
             default:
                 return Collections.emptyMap();
@@ -161,67 +163,67 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
     @Override
     public List<RuntimeClientPlugin> getClientPlugins() {
         return ListUtils.of(
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
-                             HAS_MIDDLEWARE)
-                        .servicePredicate((m, s) -> isS3(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
-                                HAS_MIDDLEWARE)
-                        .operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
-                                HAS_MIDDLEWARE)
-                        .operationPredicate(
-                                (m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
-                                        && isS3(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
-                                "WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
-                        .operationPredicate((m, s, o) -> isS3(s)
-                                            && o.getId().getName(s).equals("WriteGetObjectResponse"))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
-                                        HAS_MIDDLEWARE)
-                        .servicePredicate((m, s) -> isS3(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
-                        .operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
-                                            && isS3(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
-                                         HAS_MIDDLEWARE)
-                        .operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
-                                && isS3(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                    .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
-                        HAS_CONFIG)
-                    .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
-                    .build(),
-                 /*
-                 * BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
-                 * The second applies the middleware to bucket endpoint operations.
-                 */
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
-                                         HAS_CONFIG)
-                        .servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
-                        .build(),
-                RuntimeClientPlugin.builder()
-                        .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
-                                         HAS_MIDDLEWARE)
-                        .operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
-                                            && isS3(s)
-                                            && !isEndpointsV2Service(s)
-                                            && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
-                        .build()
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
+                    HAS_MIDDLEWARE)
+                .servicePredicate((m, s) -> isS3(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
+                    HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
+                    HAS_MIDDLEWARE)
+                .operationPredicate(
+                    (m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
+                        && isS3(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
+                    "WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> isS3(s)
+                    && o.getId().getName(s).equals("WriteGetObjectResponse"))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
+                    HAS_MIDDLEWARE)
+                .servicePredicate((m, s) -> isS3(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
+                    && isS3(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
+                    HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
+                    && isS3(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
+                    HAS_CONFIG)
+                .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
+                .build(),
+            /*
+             * BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
+             * The second applies the middleware to bucket endpoint operations.
+             */
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
+                    HAS_CONFIG)
+                .servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
+                .build(),
+            RuntimeClientPlugin.builder()
+                .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
+                    HAS_MIDDLEWARE)
+                .operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
+                    && isS3(s)
+                    && !isEndpointsV2Service(s)
+                    && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
+                .build()
         );
     }
 
@@ -232,8 +234,8 @@ private static boolean containsInputMembers(
     ) {
         OperationIndex operationIndex = OperationIndex.of(model);
         return operationIndex.getInput(operationShape)
-                .filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
-                .isPresent();
+            .filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
+            .isPresent();
     }
 
     private static boolean isS3(Shape serviceShape) {

lib/lib-storage/package.json

@@ -24,6 +24,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@aws-sdk/middleware-endpoint": "*",
     "@aws-sdk/smithy-client": "*",
     "buffer": "5.6.0",
     "events": "3.3.0",
@@ -37,6 +38,7 @@
   "devDependencies": {
     "@aws-sdk/abort-controller": "*",
     "@aws-sdk/client-s3": "*",
+    "@aws-sdk/types": "*",
     "@tsconfig/recommended": "1.0.1",
     "@types/node": "^14.11.2",
     "concurrently": "7.0.0",

lib/lib-storage/src/Upload.ts

@@ -13,8 +13,14 @@ import {
   Tag,
   UploadPartCommand,
 } from "@aws-sdk/client-s3";
+import {
+  EndpointParameterInstructionsSupplier,
+  getEndpointFromInstructions,
+  toEndpointV1,
+} from "@aws-sdk/middleware-endpoint";
 import { HttpRequest } from "@aws-sdk/protocol-http";
 import { extendedEncodeURIComponent } from "@aws-sdk/smithy-client";
+import { Endpoint } from "@aws-sdk/types";
 import { EventEmitter } from "events";
 
 import { byteLength } from "./bytelength";
@@ -101,7 +107,8 @@ export class Upload extends EventEmitter {
     this.isMultiPart = false;
     const params = { ...this.params, Body: dataPart.data };
 
-    const requestHandler = this.client.config.requestHandler;
+    const clientConfig = this.client.config;
+    const requestHandler = clientConfig.requestHandler;
     const eventEmitter: EventEmitter | null = requestHandler instanceof EventEmitter ? requestHandler : null;
     const uploadEventListener = (event: ProgressEvent) => {
       this.bytesUploadedSoFar = event.loaded;
@@ -120,14 +127,20 @@ export class Upload extends EventEmitter {
       eventEmitter.on("xhr.upload.progress", uploadEventListener);
     }
 
-    const [putResult, endpoint] = await Promise.all([
-      this.client.send(new PutObjectCommand(params)),
-      this.client.config?.endpoint?.(),
-    ]);
+    const resolved = await Promise.all([this.client.send(new PutObjectCommand(params)), clientConfig?.endpoint?.()]);
+    const putResult = resolved[0];
+    let endpoint: Endpoint = resolved[1];
+
+    if (!endpoint) {
+      endpoint = toEndpointV1(
+        await getEndpointFromInstructions(params, PutObjectCommand as EndpointParameterInstructionsSupplier, {
+          ...clientConfig,
+        })
+      );
+    }
 
     if (!endpoint) {
-      // TODO(endpointsv2): handle endpoint v2
-      throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()".');
+      throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()" nor EndpointsV2.');
     }
 
     if (eventEmitter !== null) {

packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts

@@ -5,7 +5,8 @@ import { getEndpointFromRegion } from "./utils/getEndpointFromRegion";
 
 export interface EndpointsInputConfig {
   /**
-   * The fully qualified endpoint of the webservice. This is only required when using a custom endpoint (for example, when using a local version of S3).
+   * The fully qualified endpoint of the webservice. This is only required when using 
+   * a custom endpoint (for example, when using a local version of S3).
    */
   endpoint?: string | Endpoint | Provider<Endpoint>;
 
@@ -56,7 +57,7 @@ export const resolveEndpointsConfig = <T>(
     endpoint: endpoint
       ? normalizeProvider(typeof endpoint === "string" ? urlParser(endpoint) : endpoint)
       : () => getEndpointFromRegion({ ...input, useDualstackEndpoint, useFipsEndpoint }),
-    isCustomEndpoint: endpoint ? true : false,
+    isCustomEndpoint: !!endpoint,
     useDualstackEndpoint,
   };
 };

packages/middleware-endpoint/package.json

@@ -5,6 +5,7 @@
     "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:es": "tsc -p tsconfig.es.json",
+    "build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
     "build:types": "tsc -p tsconfig.types.json",
     "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
     "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
@@ -19,6 +20,7 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@aws-sdk/protocol-http": "*",
     "@aws-sdk/signature-v4": "*",
     "@aws-sdk/types": "*",
     "@aws-sdk/util-config-provider": "*",

packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts

@@ -0,0 +1,77 @@
+import { EndpointParameters, EndpointV2, HandlerExecutionContext } from "@aws-sdk/types";
+
+import { EndpointResolvedConfig } from "../resolveEndpointConfig";
+import { EndpointParameterInstructions } from "../types";
+
+export type EndpointParameterInstructionsSupplier = Partial<{
+  getEndpointParameterInstructions(): EndpointParameterInstructions;
+}>;
+
+/**
+ * This step in the endpoint resolution process is exposed as a function
+ * to allow packages such as signers, lib-upload, etc. to get
+ * the V2 Endpoint associated to an instance of some api operation command
+ * without needing to send it or resolve its middleware stack.
+ *
+ * @private
+ * @param commandInput         - the input of the Command in question.
+ * @param instructionsSupplier - this is typically a Command constructor. A static function supplying the
+ *                               endpoint parameter instructions will exist for commands in services
+ *                               having an endpoints ruleset trait.
+ * @param clientConfig         - config of the service client.
+ * @param context              - optional context.
+ */
+export const getEndpointFromInstructions = async <
+  T extends EndpointParameters,
+  CommandInput extends Record<string, unknown>,
+  Config extends Record<string, unknown>
+>(
+  commandInput: CommandInput,
+  instructionsSupplier: EndpointParameterInstructionsSupplier,
+  clientConfig: Partial<EndpointResolvedConfig<T>> & Config,
+  context?: HandlerExecutionContext
+): Promise<EndpointV2> => {
+  const endpointParams: EndpointParameters = {};
+  const instructions: EndpointParameterInstructions =
+    (instructionsSupplier.getEndpointParameterInstructions || (() => null))() || {};
+
+  if (typeof clientConfig.endpointProvider !== "function") {
+    throw new Error("config.endpointProvider is not set.");
+  }
+
+  for (const [name, instruction] of Object.entries(instructions)) {
+    switch (instruction.type) {
+      case "staticContextParams":
+        endpointParams[name] = instruction.value;
+        break;
+      case "contextParams":
+        endpointParams[name] = commandInput[instruction.name] as string | boolean;
+        break;
+      case "clientContextParams":
+      case "builtInParams":
+        endpointParams[name] = await createConfigProvider<Config>(instruction.name, clientConfig)();
+        break;
+      default:
+        throw new Error("Unrecognized endpoint parameter instruction: " + JSON.stringify(instruction));
+    }
+  }
+
+  const endpoint: EndpointV2 = clientConfig.endpointProvider!(endpointParams as T, context);
+
+  return endpoint;
+};
+
+/**
+ * Normalize some key of the client config to an async provider.
+ * @private
+ */
+const createConfigProvider = <Config extends Record<string, unknown>>(configKey: string, config: Config) => {
+  const configProvider = async () => {
+    const configValue: unknown = config[configKey];
+    if (typeof configValue === "function") {
+      return configValue();
+    }
+    return configValue;
+  };
+  return configProvider;
+};

packages/middleware-endpoint/src/adaptors/index.ts

@@ -0,0 +1,2 @@
+export * from "./getEndpointFromInstructions";
+export * from "./toEndpointV1";