test(size-benchmark): generate benchmark and validation for package size (#3018)

Author: AllanZhengYP

.gitignore

@@ -51,4 +51,6 @@ codegen/.settings/
 codegen/*/.project
 codegen/*/.classpath
 codegen/*/.settings/
-codegen/*/bin
+codegen/*/bin
+
+benchmark/size/raw

.prettierignore

@@ -1,3 +1,5 @@
 CHANGELOG.md
 **/*/CHANGELOG.md
 .github/*
+**/*.hbs
+**/*/report.md

benchmark/size/report.md

@@ -0,0 +1,24 @@
+| Package | Version | Publish Size | browser:Webpack | browser:Rollup | browser:EsBuild |
+| :------ | :------ | :----------- | :------ | :----- | :------- |
+|@aws-sdk/abort-controller|3.40.0|41.4 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/client-s3|3.41.0|3.4 MB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/credential-provider-cognito-identity|3.41.0|116.4 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/credential-provider-env|3.40.0|45.9 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-imds|3.40.0|76.9 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-ini|3.41.0|59.9 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-node|3.41.0|60 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-process|3.40.0|46.9 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-sso|3.41.0|34.9 KB|N/A|N/A|N/A|
+|@aws-sdk/credential-provider-web-identity|3.41.0|34.4 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/credential-providers|3.41.0|78.2 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/fetch-http-handler|3.40.0|70.3 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/lib-dynamodb|3.41.0|145.1 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/lib-storage|3.41.0|66.6 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/node-http-handler|3.40.0|101.9 KB|N/A|N/A|N/A|
+|@aws-sdk/polly-request-presigner|3.41.0|36.7 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/s3-presigned-post|3.41.0|38.3 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/s3-request-presigner|3.41.0|79 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/signature-v4|3.40.0|177.6 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/signature-v4-crt|3.41.0|77.8 KB|N/A|N/A|N/A|
+|@aws-sdk/smithy-client|3.41.0|117.1 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|
+|@aws-sdk/types|3.40.0|135.2 KB|✅(5.62.1)|✅(2.59.0)|✅(0.13.12)|

package.json

@@ -34,6 +34,7 @@
     "pretest:e2e": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/{client-cloudformation,karma-credential-loader}' --include-dependencies build",
     "test:e2e": "node ./tests/e2e/index.js",
     "test:versions": "jest --config tests/versions/jest.config.js tests/versions/index.spec.ts",
+    "test:size": "cd scripts/benchmark-size/runner && yarn && ./cli.ts",
     "local-publish": "node ./scripts/verdaccio-publish/index.js",
     "lerna:version": "lerna version --exact --conventional-commits --no-push --no-git-tag-version --no-commit-hooks --loglevel silent --no-private --yes",
     "test:unit": "jest --config jest.config.js"
@@ -101,6 +102,7 @@
     "ts-jest": "^26.4.1",
     "ts-loader": "^7.0.5",
     "ts-mocha": "8.0.0",
+    "ts-node": "^10.4.0",
     "typedoc-plugin-lerna-packages": "^0.3.1",
     "typescript": "~4.3.5",
     "verdaccio": "^4.7.2",
@@ -136,4 +138,4 @@
     ],
     "**/*.{ts,js,md,json}": "prettier --write"
   }
-}
+}

scripts/benchmark-size/README.md

@@ -0,0 +1,94 @@
+# SDK Size Benchmark
+
+To avoid future issues like [#2747](https://github.com/aws/aws-sdk-js-v3/issues/2747),
+it's important to validate the v3 SDK packages' sizes are reasonable. This package
+provides a command line tool to benchmark and validate the SDK size
+metrics, including:
+
+- **publish size**: The size of the package's source code published to npm.
+- **publish files**: The count of individual files fo the package's source code
+  published to npm.
+- **Webpack compatibility**: Whether users can run Webpack command successfully
+  with minimal configuration, and version of Webpack tool.
+- **Rollup compatibility**: Whether users can run Rollup command successfully
+  with minimal configuration, and version of Rollup tool.
+- **Esbuild compatibility**: Whether users can run Esbuild command successfully
+  with minimal configuration, and version of Esbuild tool.
+
+By default, the report Markdown file is generated under [report.md](report)
+
+## How to use it
+
+By default, the tool doesn't require any options, but it offers some options in
+case you need to customize the behavior. You can see the full descriptions by
+running:
+
+```console
+# from project root
+yarn test:size --help
+```
+
+here's the options description:
+
+```console
+Options:
+  --version           Show version number                              [boolean]
+  --since             Run the size benchmark on changed package since last
+                      release or main branch
+              [string] [choices: "all", "last_release", "main"] [default: "all"]
+  --scopeConfigPath
+  [string] [default: "/path/to/project/root/scripts/benchmark-size/scope.json"]
+  --limitConfigPath
+  [string] [default: "/path/to/project/root/scripts/benchmark-size/limit.json"]
+  --skipLocalPublish  Skip publishing the packages locally. You can skip it if
+                      you didn't change any packages since last execution
+                                                      [boolean] [default: false]
+  --rawOutputPath [string] [default: "/path/to/project/root/benchmark/size/raw"]
+  --reportPath                                                [string] [default:
+        "/path/to/project/root/benchmark/size/report.md"]
+  --skipRawOutput     Whether to generate the raw benchmark data to configured
+                      path                            [boolean] [default: false]
+  --help              Show help                                        [boolean]
+```
+
+## Important Configuration Files
+
+The maintainers only need to test the packages that is public for customer to
+consume publicly like all service clients and some utility packages like
+presigner. The other internal packages are tested as dependencies of those
+public packages. There are 2 config files that maintainers need to update update
+to make sure the tool runs on all packages it suppose to run and validate them
+with reasonable limit:
+
+- [`scope.json`][scope-json]: Define what packages are considered as public, so
+  they will exist as an entry in the report. You can use wild card in the `package`
+  entry. For packages that requires peer dependencies, you can specify
+  `dependencies` list. You can skip running the browser bundlers test if the package
+  is Node.js runtime only by setting the `skipBundlerTests` boolean entry.
+- [`limit.json`][limit-json]: Define how to validate the packages' size. For each
+  package you can validate either of the metrics the tool generates: `publishSize`
+  and `publishFiles`. For each metrics, you can validate them by `limit` -- the
+  absolute value, or `hike` -- the percentage of increase since recent value.
+
+## How does it work
+
+The command works in following process:
+
+1. Detect the packages that changed since last release, thus requires validating
+   their size metrics before next release.
+1. Validate they have been built locally and release to a local NPM registry
+1. For each changed package, creating a temporary project with [the project templates][templates]
+   The templates generation context is exactly defined in [`scope.json`][scope-json].
+1. Run NPM install from the local registry and calculate the npm size metrics.
+1. Run Webpack, Rollup, and Esbuild bundlers on the temporary project and calculate
+   the bundle size for each of them.
+1. Clear temporary projects.
+1. Update the [report.md][report]. Validating them according to [`limit.json`][limit-json]
+
+[Prior arts][prior-arts] referred during the implementation.
+
+[report]: ../../benchmark/size/report.md
+[scope-json]: ./scope.json
+[limit-json]: ./limit.json
+[templates]: ./templates
+[prior-arts]: https://github.com/styfle/packagephobia#how-is-this-different

scripts/benchmark-size/limit.json

@@ -0,0 +1,8 @@
+{
+  "default": {
+    "publishSize": { "limit": "1 mb", "hike": "10 %" }
+  },
+  "@aws-sdk/client-ec2": {
+    "publishSize": { "limit": "10 mb" }
+  }
+}

scripts/benchmark-size/runner/.gitignore

@@ -0,0 +1 @@
+node_modules

scripts/benchmark-size/runner/calculate-size/bundlers-size.ts

@@ -0,0 +1,73 @@
+import commonjsPlugin from "@rollup/plugin-commonjs";
+import jsonPlugin from "@rollup/plugin-json";
+import resolvePlugin from "@rollup/plugin-node-resolve";
+import { build as esbuild } from "esbuild";
+import { lstat } from "fs-extra";
+import { join } from "path";
+import { rollup } from "rollup";
+import { terser as terserPlugin } from "rollup-plugin-terser";
+import { promisify } from "util";
+import webpack from "webpack";
+
+import type { SizeReportContext } from "../index";
+
+export interface BundlerSizeReportContext extends SizeReportContext {
+  projectDir: string;
+  entryPoint: string;
+}
+
+export const getWebpackSize = async (context: BundlerSizeReportContext): Promise<number> => {
+  const webpackOutputDir = join(context.projectDir, "webpack-dist");
+  const webpackInstance = webpack({
+    entry: context.entryPoint,
+    output: { path: webpackOutputDir, filename: "index.js" },
+  });
+  const run = promisify(webpackInstance.run);
+  const webpackStats = await run.call(webpackInstance);
+  if (webpackStats.hasErrors()) {
+    throw new Error(webpackStats.toString());
+  }
+  return (await lstat(join(webpackOutputDir, "index.js"))).size;
+};
+
+export const getRollupSize = async (context: BundlerSizeReportContext): Promise<number> => {
+  const rollupOutputDir = join(context.projectDir, "rollup-dist");
+  const bundle = await rollup({
+    input: context.entryPoint,
+    treeshake: true,
+    plugins: [
+      resolvePlugin({
+        // Unlike webpack, nodejs plugin provides built-in polyfills for "events" in packages like lib-storage
+        preferBuiltins: true,
+      }),
+      commonjsPlugin(),
+      jsonPlugin(),
+      terserPlugin(),
+    ],
+    // Omit circular dependency warning: Circular dependency: node_modules/@aws-crypto/crc32/build/index.js ->
+    // node_modules/@aws-crypto/crc32/build/aws_crc32.js -> node_modules/@aws-crypto/crc32/build/index.js
+    onwarn: (warning, defaultHandler) => {
+      if (warning.code !== "CIRCULAR_DEPENDENCY") {
+        defaultHandler(warning);
+      }
+    },
+  });
+  await bundle.write({
+    file: join(rollupOutputDir, "index.js"),
+  });
+  await bundle.close();
+  return (await lstat(join(rollupOutputDir, "index.js"))).size;
+};
+
+export const getEsbuildSize = async (context: BundlerSizeReportContext): Promise<number> => {
+  const esbuildOutputDir = join(context.projectDir, "esbuild-dist");
+  await esbuild({
+    entryPoints: [context.entryPoint],
+    outbase: context.projectDir,
+    outdir: esbuildOutputDir,
+    bundle: true,
+    minify: true,
+    treeShaking: true,
+  });
+  return (await lstat(join(esbuildOutputDir, "index.js"))).size;
+};

scripts/benchmark-size/runner/calculate-size/generate-project.ts

@@ -0,0 +1,70 @@
+import exec from "execa";
+import { promises as fsPromise } from "fs";
+import { join } from "path";
+import prettier from "prettier";
+
+import { PackageContext } from "../load-test-scope";
+import type { PackageSizeReportOptions } from "./index";
+
+export const generateProject = async (projectDir: string, options: PackageSizeReportOptions) => {
+  const peerDependencies = await getPeerDependencies(options);
+  const contextWithPeerDep: PackageContext = {
+    ...options.packageContext,
+    dependencies: [...peerDependencies, ...(options.packageContext?.dependencies ?? [])],
+  };
+  // console.error("CONTEXT||||||||", contextWithPeerDep);
+  for (const [name, template] of Object.entries(options.templates)) {
+    const filePath = join(projectDir, name);
+    const file = prettier.format(template(contextWithPeerDep), {
+      filepath: filePath,
+    });
+    // console.error("FILE|||||||, ", file);
+    await fsPromise.writeFile(filePath, file);
+  }
+
+  await exec(
+    "npm",
+    [
+      "install",
+      "--cache",
+      options.npmCacheDir,
+      "--no-audit",
+      "--no-update-notifier",
+      "--no-package-lock",
+      "--no-progress",
+      "--production",
+      "--silent",
+    ],
+    {
+      cwd: projectDir,
+      env: {
+        npm_config_registry: options.localRegistry,
+      },
+    }
+  );
+};
+
+type PeerDependencies = { name: string; version: string }[];
+
+/**
+ * Reads the peer dependencies from the package.json. These dependencies will be added to the
+ * generated project's dependencies.
+ */
+const getPeerDependencies = async (options: PackageSizeReportOptions): Promise<PeerDependencies> => {
+  // console.error("||||||||NAME: ", options.packageName);
+  const packageInfo = options.workspacePackages.find((pkg) => pkg.name === options.packageName);
+  if (!packageInfo) {
+    throw new Error(`Cannot find package ${options.packageName} from the workspace`);
+  }
+  const packageJsonPath = join(packageInfo.location, "package.json");
+  let peerDependencies: PeerDependencies;
+  try {
+    peerDependencies = JSON.parse(await fsPromise.readFile(packageJsonPath, "utf-8"))["peerDependencies"] ?? {};
+  } catch (e) {
+    throw new Error(`Cannot load the package.json file from ${packageJsonPath}`);
+  }
+  // Change the peer dependencies' version to "ci", referring to the current local package.
+  // console.error("|||||PD: ", peerDependencies);
+  // console.error("|||||PI: ", packageInfo);
+  return Object.entries(peerDependencies).map(([key]) => ({ name: key, version: "ci" }));
+};

scripts/benchmark-size/runner/calculate-size/index.ts

@@ -0,0 +1,67 @@
+import { promises as fsPromise } from "fs";
+import { ListrContext, ListrTaskWrapper } from "listr2";
+import { join } from "path";
+
+import { SizeReportContext } from "../index";
+import { PackageContext } from "../load-test-scope";
+import { getEsbuildSize, getRollupSize, getWebpackSize } from "./bundlers-size";
+import { generateProject } from "./generate-project";
+import { calculateNpmSize } from "./npm-size";
+
+export interface PackageSizeReportOptions extends SizeReportContext {
+  packageName: string;
+  packageContext: PackageContext;
+}
+
+export interface PackageSizeReportOutput {
+  name: string;
+  version: string;
+  installSize: number;
+  publishSize: number;
+  webpackSize: number | undefined;
+  esbuildSize: number | undefined;
+  rollupSize: number | undefined;
+}
+
+export const getPackageSizeReportRunner =
+  (options: PackageSizeReportOptions) => async (context: ListrContext, task: ListrTaskWrapper<ListrContext, any>) => {
+    task.output = "preparing...";
+    const projectDir = join(options.tmpDir, options.packageName.replace("/", "_"));
+    await fsPromise.rmdir(projectDir, { recursive: true });
+    await fsPromise.mkdir(projectDir);
+    const entryPoint = join(projectDir, "index.js");
+    const bundlersContext = { ...options, entryPoint, projectDir };
+
+    task.output = "generating project and installing dependencies";
+    await generateProject(projectDir, options);
+
+    task.output = "calculating npm size";
+    const npmSizeResult = calculateNpmSize(projectDir, options.packageName);
+
+    const skipBundlerTests = bundlersContext.packageContext.skipBundlerTests;
+
+    task.output = "calculating webpack 5 full bundle size";
+    const webpackSize = skipBundlerTests ? undefined : await getWebpackSize(bundlersContext);
+
+    task.output = "calculating rollup full bundle size";
+    const rollupSize = skipBundlerTests ? undefined : await getRollupSize(bundlersContext);
+
+    task.output = "calculating esbuild full bundle size";
+    const esbuildSize = skipBundlerTests ? undefined : await getEsbuildSize(bundlersContext);
+
+    task.output = "output results";
+    const packageVersion = JSON.parse(
+      await fsPromise.readFile(
+        join(options.workspacePackages.filter((pkg) => pkg.name === options.packageName)[0].location, "package.json"),
+        "utf8"
+      )
+    ).version;
+    options.output.push({
+      name: options.packageName,
+      version: packageVersion,
+      ...npmSizeResult,
+      webpackSize,
+      esbuildSize,
+      rollupSize,
+    });
+  };