test: deploy integration test stack automatically and new test scripts (#2944)

* chore: add ci script for building changed packages

* chore: reorder ci scripts

* chore: skip integ test if no client changes

* test(e2e): deploy integration test stack automatically

* test(e2e): use node10 syntax

* test(e2e): exit script with 0 code after complete

This is to fix the segment fault issue in the Node.js 12 e2e test
in CodeBuild runtime after returning. This issue cannot be reproduced
in other Node.js version or locally.

Author: AllanZhengYP

package.json

@@ -10,18 +10,21 @@
     "generate-clients": "node ./scripts/generate-clients",
     "generate:clients:generic": "node ./scripts/generate-clients/generic",
     "bootstrap": "yarn",
+    "bootstrap:ci": "yarn install --frozen-lockfile",
     "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean",
     "clear-build-cache": "rimraf ./packages/*/dist ./clients/*/dist ./lib/*/dist ./private/*/dist",
     "clear-build-info": "rimraf ./packages/**/*.tsbuildinfo ./clients/**/*.tsbuildinfo  ./lib/**/*.tsbuildinfo ./private/**/*.tsbuildinfo",
     "remove-documentation": "rimraf ./docs",
     "build:clients:generic": "lerna run --scope '@aws-sdk/aws-echo-service' --include-dependencies build",
+    "build:packages": "lerna run build --ignore '@aws-sdk/client-*' --ignore '@aws-sdk/aws-*' --ignore '@aws-sdk/lib-*' --include-dependencies",
     "build:crypto-dependencies": "lerna run --scope '@aws-sdk/{types,util-utf8-browser,util-locate-window,hash-node}' --include-dependencies build",
     "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-protocoltests-*' --include-dependencies build",
     "build:server-protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/*-server' --include-dependencies build",
+    "build:clients:since:release": "yarn ci:build:packages && lerna run build --since --scope '@aws-sdk/client-*' --scope '@aws-sdk/lib-*'",
     "build:all": "yarn build:crypto-dependencies && lerna run build",
     "build-documentation": "yarn remove-documentation && typedoc",
     "pretest:all": "yarn build:all",
-    "test:all": "jest --coverage --passWithNoTests && lerna run test --scope '@aws-sdk/{fetch-http-handler,hash-blob-browser}'",
+    "test:all": "jest --coverage --passWithNoTests && lerna run test --scope '@aws-sdk/{fetch-http-handler,hash-blob-browser}' && yarn test:versions",
     "test:functional": "jest --config tests/functional/jest.config.js",
     "test:integration:legacy": "cucumber-js --fail-fast",
     "test:integration:legacy:since:release": "./tests/integ-legacy/index.js",
@@ -32,7 +35,8 @@
     "test:e2e": "node ./tests/e2e/index.js",
     "test:versions": "jest --config tests/versions/jest.config.js tests/versions/index.spec.ts",
     "local-publish": "node ./scripts/verdaccio-publish/index.js",
-    "lerna:version": "lerna version --exact --conventional-commits --no-push --no-git-tag-version --no-commit-hooks --loglevel silent --no-private --yes"
+    "lerna:version": "lerna version --exact --conventional-commits --no-push --no-git-tag-version --no-commit-hooks --loglevel silent --no-private --yes",
+    "test:unit": "jest --config jest.config.js"
   },
   "repository": {
     "type": "git",

tests/e2e/ensure-test-stack.js

@@ -0,0 +1,67 @@
+const {
+  CreateChangeSetCommand,
+  DescribeStacksCommand,
+  waitUntilChangeSetCreateComplete,
+  Capability,
+  ExecuteChangeSetCommand,
+  waitUntilStackUpdateComplete,
+  waitUntilStackCreateComplete,
+  DescribeChangeSetCommand,
+} = require("../../clients/client-cloudformation");
+
+/**
+ * Deploy the integration test stack if it does not exist. Update the
+ * stack if there's a changeset. There should be a high level library for it
+ * really.
+ */
+exports.ensureTestStack = async (client, stackName, templateBody) => {
+  let hasExistingStack = false;
+  try {
+    await client.send(new DescribeStacksCommand({ StackName: stackName }));
+    hasExistingStack = true;
+  } catch (e) {
+    if ((e.message || "").endsWith("does not exist")) {
+      hasExistingStack = false;
+    } else {
+      throw e;
+    }
+  }
+
+  const { Id } = await client.send(
+    new CreateChangeSetCommand({
+      StackName: stackName,
+      ChangeSetType: hasExistingStack ? "UPDATE" : "CREATE",
+      ChangeSetName: `${stackName}ChangeSet`,
+      TemplateBody: templateBody,
+      Capabilities: [Capability.CAPABILITY_IAM],
+    })
+  );
+
+  try {
+    // TODO: Feature Request: return the last describe response from the waiter result, so we can inspect failure reasons.
+    await waitUntilChangeSetCreateComplete({ client }, { ChangeSetName: Id });
+  } catch (e) {
+    const { Status, StatusReason } = await client.send(
+      new DescribeChangeSetCommand({
+        StackName: stackName,
+        ChangeSetName: Id,
+      })
+    );
+    if (Status === "FAILED" && StatusReason.includes("The submitted information didn't contain changes")) {
+      return;
+    }
+    throw e;
+  }
+
+  await client.send(
+    new ExecuteChangeSetCommand({
+      ChangeSetName: Id,
+      StackName: stackName,
+    })
+  );
+  if (hasExistingStack) {
+    await waitUntilStackUpdateComplete({ client }, { StackName: stackName });
+  } else {
+    await waitUntilStackCreateComplete({ client }, { StackName: stackName });
+  }
+};

tests/e2e/get-integ-test-resources.js

@@ -1,13 +1,24 @@
+const { readFileSync } = require("fs");
+const { join } = require("path");
 const { CloudFormationClient, DescribeStackResourcesCommand } = require("../../clients/client-cloudformation");
+const { ensureTestStack } = require("./ensure-test-stack");
 
 exports.getIntegTestResources = async () => {
-  const client = new CloudFormationClient({});
+  const client = new CloudFormationClient({ logger: console });
   const region = await client.config.region();
+  const stackName = "SdkReleaseV3IntegTestResourcesStack";
 
+  await ensureTestStack(
+    client,
+    stackName,
+    readFileSync(join(__dirname, "IntegTestResourcesStack.template.json"), { encoding: "utf-8" })
+  );
   const { StackResources: stackResources } = await client.send(
-    new DescribeStackResourcesCommand({ StackName: "SdkReleaseV3IntegTestResourcesStack" })
+    new DescribeStackResourcesCommand({ StackName: stackName })
   );
 
+  console.log(`${stackName} Stack Resources: `, stackResources);
+
   const identityPoolId = stackResources.filter((resource) => resource.ResourceType === "AWS::Cognito::IdentityPool")[0]
     .PhysicalResourceId;
 

tests/e2e/index.js

@@ -5,6 +5,7 @@ const run = async () => {
   try {
     const integTestResourcesEnv = await getIntegTestResources();
     await runE2ETests(integTestResourcesEnv);
+    process.exit(0);
   } catch (e) {
     console.error(e);
     process.exit(1);

tests/e2e/run-e2e-tests.js

@@ -1,4 +1,4 @@
-const { execSync, spawn } = require("child_process");
+const { execSync } = require("child_process");
 const { join } = require("path");
 const { readFileSync, existsSync } = require("fs");
 const { spawnPromise } = require("./spawn-promise");
@@ -17,9 +17,12 @@ exports.runE2ETests = async (resourcesEnv) => {
    /path/to/package:@aws-sdk/client-acm-pca:1.0.0-gamma.3
    /path/to/package:@aws-sdk/client-acm:1.0.0-gamma.3
    */
-  const changedPackagesRecord = execSync(
-    "./node_modules/.bin/lerna  changed --all --parseable --long --loglevel silent"
-  );
+  let changedPackagesRecord = "";
+  try {
+    changedPackagesRecord = execSync("./node_modules/.bin/lerna changed --all --parseable --long --loglevel silent");
+  } catch (e) {
+    // Swallow error because Lerna throws if no package changes.
+  }
   // Get array for changed package's path
   const changedPackages = changedPackagesRecord
     .toString()
@@ -28,7 +31,7 @@ exports.runE2ETests = async (resourcesEnv) => {
     .map((record) => record.split(":").slice(0, 2));
   console.log(changedPackages);
   const packagesToTest = changedPackages.filter((changedPackage) => hasE2Etest(changedPackage[0]));
-  if (packagesToTest?.length === 0) {
+  if (packagesToTest.length === 0) {
     console.log("no changed package contains e2e test.");
     return;
   }

tests/integ-legacy/index.js

@@ -17,14 +17,24 @@ const allTags = execSync(`grep -h ^@ ${join(FEATURES_FOLDER, "**", "*.feature")}
 
 console.info(`Looking for changed clients that has the legacy integration test tag: ${allTags}`);
 
-const changedPackages = execSync(`${join(ROOT_BIN, "lerna")} changed`, execOptions).split("\n");
+let changedPackages = [];
+try {
+  changedPackages = execSync(`${join(ROOT_BIN, "lerna")} changed`, execOptions).split("\n");
+} catch (e) {
+  // Swallow error because Lerna throws if no package changes.
+}
 const changedPackageTags = changedPackages
   .map((name) => name.replace("@aws-sdk/client-", ""))
   .map((name) => name.replace("-"))
   .map((name) => `@${name}`);
 
 const tagsToTest = changedPackageTags.filter((tag) => allTags.includes(tag));
 
+if (tagsToTest.length === 0) {
+  console.info("No clients with integration test cases has changed since last release.");
+  return;
+}
+
 // Cucumber requires cwd to contain the test cases.
 const command = `${join("node_modules", ".bin", "cucumber-js")}`;
 const args = ["--fail-fast", "-t", `"${tagsToTest.join(" or ")}"`];