build: prepare NPM packages for use in Bazel-based integration tests

Updates the build tooling to prepare NPM packages from the project
`package.json` to be accessible to the Bazel-based integration tests.

The idea is to build tarballs for all NPM packages which can be mapped
into Bazel-based integration tests. This allows us to only have a single
dependency on e.g. the CLI, instead of having to maintain/update
multiple versions of the CLI and devkit (as an example).

Additionally, we will expose tarballs for the first-party built NPM
packages like the CDK or Angular Material.

Lastly, we will populate the `0.0.0-PLACEHOLDER` with an actual version
when stamping is disabled. This allows us to use the first-party built
NPM packages in tests with `yarn`. Yarn would otherwise complain about
an invalid version.

Author: devversion

.gitignore

@@ -10,6 +10,9 @@
 node_modules
 /bower_components
 
+# Angular CLI project cache
+.angular/
+
 # Dart
 /.pub
 /.packages

WORKSPACE

@@ -34,6 +34,19 @@ http_archive(
     ],
 )
 
+http_archive(
+    name = "rules_pkg",
+    sha256 = "a89e203d3cf264e564fcb96b6e06dd70bc0557356eb48400ce4b5d97c2c3720d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.5.1/rules_pkg-0.5.1.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.5.1/rules_pkg-0.5.1.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 
 bazel_skylib_workspace()
@@ -47,13 +60,17 @@ node_repositories(
     package_json = ["//:package.json"],
 )
 
+load("//tools:integration.bzl", "create_npm_package_archive_build_file")
+
 yarn_install(
     name = "npm",
     # We add the postinstall patches file here so that Yarn will rerun whenever
     # the file is modified.
     data = [
         "//:tools/postinstall/apply-patches.js",
     ],
+    # Add archive targets for some NPM packages that are needed in integration tests.
+    manual_build_file_contents = create_npm_package_archive_build_file(),
     package_json = "//:package.json",
     quiet = False,
     yarn_lock = "//:yarn.lock",

packages.bzl

@@ -6,10 +6,10 @@ MDC_PACKAGE_VERSION = "13.0.0-canary.860ad06a1.0"
 TSLIB_PACKAGE_VERSION = "^2.3.0"
 RXJS_PACKAGE_VERSION = "^6.5.3 || ^7.0.0"
 
-# Each placer holder is used to stamp versions during the build process, replacing the key with it's
+# Each placeholder is used to stamp versions during the build process, replacing the key with it's
 # value pair. These replacements occur during building of `npm_package` and `ng_package` stamping in
 # the peer dependencies and versions, primarily in `package.json`s.
-VERSION_PLACEHOLDER_REPLACEMENTS = {
+NPM_PACKAGE_SUBSTITUTIONS = {
     # Version of `material-components-web`
     "0.0.0-MDC": MDC_PACKAGE_VERSION,
     # Version of `@angular/core`
@@ -22,6 +22,11 @@ VERSION_PLACEHOLDER_REPLACEMENTS = {
     "0.0.0-RXJS": RXJS_PACKAGE_VERSION,
 }
 
+TEST_NPM_PACKAGE_SUBSTITUTIONS = dict(NPM_PACKAGE_SUBSTITUTIONS, **{
+    # When building NPM packages for tests, we use `0.0.0` for the version placeholder.
+    "0.0.0-PLACEHOLDER": "0.0.0",
+})
+
 # List of MDC packages.
 MDC_PACKAGES = [
     "@material/animation",

tools/BUILD.bazel

@@ -13,6 +13,12 @@ config_setting(
     },
 )
 
+# Detect if the build is running with stamping enabled.
+config_setting(
+    name = "stamp",
+    values = {"stamp": "true"},
+)
+
 # This file generates the `angular_ivy_enabled.js` file, which exports a truthy value
 # whether Ivy should be enabled based on the `--defined=angular_ivy_enabled` value
 # so runtime can detect which mode it is running in.

tools/defaults.bzl

@@ -1,22 +1,30 @@
 # Re-export of Bazel rules with repository-wide defaults
 
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@build_bazel_rules_nodejs//:index.bzl", _pkg_npm = "pkg_npm")
 load("@io_bazel_rules_sass//:defs.bzl", _npm_sass_library = "npm_sass_library", _sass_binary = "sass_binary", _sass_library = "sass_library")
 load("@npm//@angular/bazel:index.bzl", _ng_module = "ng_module", _ng_package = "ng_package")
+load("@npm//@angular/dev-infra-private/bazel/integration:index.bzl", _integration_test = "integration_test")
 load("@npm//@bazel/jasmine:index.bzl", _jasmine_node_test = "jasmine_node_test")
 load("@npm//@bazel/concatjs:index.bzl", _karma_web_test = "karma_web_test", _karma_web_test_suite = "karma_web_test_suite")
 load("@npm//@bazel/protractor:index.bzl", _protractor_web_test_suite = "protractor_web_test_suite")
 load("@npm//@bazel/typescript:index.bzl", _ts_library = "ts_library")
-load("//:packages.bzl", "VERSION_PLACEHOLDER_REPLACEMENTS")
+load("//:packages.bzl", "NPM_PACKAGE_SUBSTITUTIONS", "TEST_NPM_PACKAGE_SUBSTITUTIONS")
 load("//:pkg-externals.bzl", "PKG_EXTERNALS")
 load("//tools/markdown-to-html:index.bzl", _markdown_to_html = "markdown_to_html")
 load("//tools/spec-bundling:index.bzl", "spec_bundle")
 
 _DEFAULT_TSCONFIG_BUILD = "//src:bazel-tsconfig-build.json"
 _DEFAULT_TSCONFIG_TEST = "//src:tsconfig-test"
 
+npmPackageSubstitutions = select({
+    "//tools:stamp": NPM_PACKAGE_SUBSTITUTIONS,
+    "//conditions:default": TEST_NPM_PACKAGE_SUBSTITUTIONS,
+})
+
 # Re-exports to simplify build file load statements
 markdown_to_html = _markdown_to_html
+integration_test = _integration_test
 
 def _compute_module_name(testonly):
     current_pkg = native.package_name()
@@ -147,7 +155,7 @@ def ng_module(
         **kwargs
     )
 
-def ng_package(name, data = [], deps = [], externals = PKG_EXTERNALS, readme_md = None, **kwargs):
+def ng_package(name, data = [], deps = [], externals = PKG_EXTERNALS, readme_md = None, visibility = None, **kwargs):
     # If no readme file has been specified explicitly, use the default readme for
     # release packages from "src/README.md".
     if not readme_md:
@@ -183,12 +191,24 @@ def ng_package(name, data = [], deps = [], externals = PKG_EXTERNALS, readme_md
         package_name = None,
         validate = False,
         readme_md = readme_md,
-        substitutions = VERSION_PLACEHOLDER_REPLACEMENTS,
+        substitutions = npmPackageSubstitutions,
+        visibility = visibility,
         **kwargs
     )
 
-def pkg_npm(**kwargs):
+    pkg_tar(
+        name = name + "_archive",
+        srcs = [":%s" % name],
+        extension = "tar.gz",
+        strip_prefix = "./%s" % name,
+        # Target should not build on CI unless it is explicitly requested.
+        tags = ["manual"],
+        visibility = visibility,
+    )
+
+def pkg_npm(name, visibility = None, **kwargs):
     _pkg_npm(
+        name = name,
         # We never set a `package_name` for NPM packages, neither do we enable validation.
         # This is necessary because the source targets of the NPM packages all have
         # package names set and setting a similar `package_name` on the NPM package would
@@ -204,10 +224,21 @@ def pkg_npm(**kwargs):
         # https://github.com/bazelbuild/rules_nodejs/issues/2810.
         package_name = None,
         validate = False,
-        substitutions = VERSION_PLACEHOLDER_REPLACEMENTS,
+        substitutions = npmPackageSubstitutions,
+        visibility = visibility,
         **kwargs
     )
 
+    pkg_tar(
+        name = name + "_archive",
+        srcs = [":%s" % name],
+        extension = "tar.gz",
+        strip_prefix = "./%s" % name,
+        # Target should not build on CI unless it is explicitly requested.
+        tags = ["manual"],
+        visibility = visibility,
+    )
+
 def jasmine_node_test(**kwargs):
     kwargs["templated_args"] = ["--bazel_patch_module_resolver"] + kwargs.get("templated_args", [])
     _jasmine_node_test(**kwargs)
@@ -306,6 +337,23 @@ def protractor_web_test_suite(name, deps, **kwargs):
         **kwargs
     )
 
+def node_integration_test(data = [], tool_mappings = {}, **kwargs):
+    """Macro for defining an integration test with `node` and `yarn` being
+      declared as global tools."""
+
+    integration_test(
+        data = data + [
+            # The Yarn files also need to be part of the integration test as runfiles
+            # because the `yarn_bin` target is not a self-contained standalone binary.
+            "@nodejs//:yarn_files",
+        ],
+        tool_mappings = dict(tool_mappings, **{
+            "@nodejs//:yarn_bin": "yarn",
+            "@nodejs//:node_bin": "node",
+        }),
+        **kwargs
+    )
+
 def ng_web_test_suite(deps = [], static_css = [], exclude_init_script = False, **kwargs):
     bootstrap = [
         # This matches the ZoneJS bundles used in default CLI projects. See:

tools/integration.bzl

@@ -0,0 +1,46 @@
+load("//:packages.bzl", "ANGULAR_PACKAGES")
+
+"""File which manages the NPM packages from the workspace `@npm` repository which
+  should be available to integration tests"""
+
+def _get_archive_label_of_package(package_name):
+    return package_name.replace("/", "_").replace("@", "") + "_archive"
+
+INTEGRATION_TEST_PACKAGES = [pkg.module_name for pkg in ANGULAR_PACKAGES] + [
+    "@angular/cli",
+    "@angular/compiler-cli",
+    "@angular-devkit/build-angular",
+    "typescript",
+    "rxjs",
+]
+
+CLI_PROJECT_MAPPINGS = {
+    "@npm//:%s" % _get_archive_label_of_package(pkg): pkg
+    for pkg in INTEGRATION_TEST_PACKAGES
+}
+
+def create_npm_package_archive_build_file():
+    """Creates the contents of a `BUILD.bazel` file for exposing NPM package tarballs
+      for the integration test packages configured in the constant.
+
+      The `BUILD.bazel` file contents are supposed to be placed into the `@npm//`
+      workspace top-level BUILD file. This is necessary because all files of a NPM
+      package are not accessible outside from the `@npm//` workspace.
+      """
+
+    result = """load("@rules_pkg//:pkg.bzl", "pkg_tar")"""
+
+    for pkg in INTEGRATION_TEST_PACKAGES:
+        label_name = _get_archive_label_of_package(pkg)
+        last_segment = pkg.split("/")[-1]
+
+        result += """
+pkg_tar(
+    name = "{label_name}",
+    srcs = ["//{name}:{last_segment}__all_files"],
+    extension = "tar.gz",
+    strip_prefix = "/external/npm/node_modules/{name}",
+    tags = ["manual"],
+)""".format(name = pkg, label_name = label_name, last_segment = last_segment)
+
+    return result