build: run bazel tests on CI and generate bazel build dist/npm_bazel folder (#1754)

Next steps after this are testing of all local workflows and cut-over to a bazel built release. Will co-ordinate with Angular team on this.

(cherry picked from commit 4007657)

Author: gregmagolan

.bazelrc

@@ -1,5 +1,27 @@
-# Enable debugging tests with --config=debug
-test:debug --test_arg=--node_options=--inspect-brk --test_output=streamed --test_strategy=exclusive --test_timeout=9999 --nocache_test_results
+# ======================================================================================================================
+# Support for debugging Node.js tests
+# Use `--config=debug` to enable these settings
+
+# Use bazel run with `--config=debug` to turn on the NodeJS inspector agent.
+# The node process will break before user code starts and wait for the debugger to connect.
+# Pass the --inspect-brk option to all tests which enables the node inspector agent.
+# See https://nodejs.org/de/docs/guides/debugging-getting-started/#command-line-options for more details.
+run:debug -- --node_options=--inspect-brk
+
+# Stream stdout/stderr output from each test in real-time.
+# See https://docs.bazel.build/versions/master/user-manual.html#flag--test_output for more details.
+test:debug --test_output=streamed
+
+# Run one test at a time.
+test:debug --test_strategy=exclusive
+
+# Prevent long running tests from timing out.
+# See https://docs.bazel.build/versions/master/user-manual.html#flag--test_timeout for more details.
+test:debug --test_timeout=9999
+
+# Always run tests even if they have cached results.
+test:debug --nocache_test_results
+# ======================================================================================================================
 
 ###############################
 # Filesystem interactions     #

.circleci/config.yml

@@ -27,25 +27,12 @@ jobs:
       - run: scripts/format.sh
       - run: scripts/build.sh << parameters.dependency_type >>
       - run: scripts/test.sh
-  bazel-build-and-test:
-    docker:
-      - image: cimg/node:14.18.3@sha256:30583ce79b08d318b1fc5e17a613d21ef960091e5442b06eb49c56fa180a3b30
-    steps:
-      - checkout
-      - node/install-packages:
-          pkg-manager: yarn
-      # Disable bazel build & test temporarily
-      - run: yarn bazel info
 workflows:
     build-and-test:
       jobs:
         - build-and-test:
             name: build-and-test
             dependency_type: "package.json"
-    bazel-build-and-test:
-      jobs:
-        - bazel-build-and-test:
-            name: bazel-build-and-test
     buid-and-test-against-builds:
       jobs:
         - build-and-test:

.gitignore

@@ -1,4 +1,3 @@
-.npmrc
 .vscode-test
 .DS_Store
 node_modules

.npmrc

@@ -0,0 +1,8 @@
+# Disabling pnpm [hoisting](https://pnpm.io/npmrc#hoist) by setting `hoist=false` is recommended on
+# projects using rules_js so that pnpm outside of Bazel lays out a node_modules tree similar to what
+# rules_js lays out under Bazel (without a hidden node_modules/.pnpm/node_modules)
+hoist=false
+
+# Allow for missing peer dependencies on pnpm import.
+# See https://github.com/aspect-build/rules_js/pull/445 for more context.
+strict-peer-dependencies=false

BUILD.bazel

@@ -1,8 +1,10 @@
 load("@aspect_rules_js//js:defs.bzl", "js_library")
+load("@aspect_rules_js//js/private:expand_template.bzl", "expand_template")
 load("@aspect_rules_js//npm:defs.bzl", "npm_package")
 load("@aspect_rules_ts//ts:defs.bzl", "ts_config")
 load("@npm//:defs.bzl", "npm_link_all_packages")
 load("@npm//:vsce/package_json.bzl", vsce_bin = "bin")
+load(":version.bzl", "VERSION")
 
 npm_link_all_packages(name = "node_modules")
 
@@ -18,10 +20,20 @@ ts_config(
     visibility = ["//visibility:public"]
 )
 
+expand_template(
+    name = "package_json_expanded",
+    template = "package.json",
+    out = "package_expanded.json",
+    substitutions = {
+        "0.0.0-PLACEHOLDER": VERSION,
+        "./dist/client/src/extension": "./index",
+    }
+)
+
 npm_package(
-    name = "npm",
+    name = "vsix_sandbox",
     srcs = [
-        "package.json",
+        "package_expanded.json",
         "angular.png",
         "CHANGELOG.md",
         "README.md",
@@ -57,22 +69,37 @@ npm_package(
         "**/*.map",
     ],
     replace_prefixes = {
+        "package_expanded.json": "package.json",
         "client/": "",
         "syntaxes/src/": "syntaxes/",
     },
-    visibility = ["//integration:__subpackages__"],
 )
 
 vsce_bin.vsce(
     name = "vsix",
     srcs = [
-        ":npm",
+        ":vsix_sandbox",
     ],
-    outs = ["ng-template.vsix"],
-    chdir = "npm",
+    outs = ["ng-template-{}.vsix".format(VERSION)],
+    chdir = "vsix_sandbox",
     args = [
         "package",
         "-o",
-        "../ng-template.vsix"
+        "../ng-template-{}.vsix".format(VERSION)
+    ],
+    # vsce requires npm on the PATH; we can get this from the Bazel rules_nodejs but it is not
+    # included by default in rules_js binary rules so we include it here explicitly
+    include_npm = True,
+)
+
+npm_package(
+    name = "npm",
+    srcs = [
+        ":vsix_sandbox",
+        ":vsix",
+    ],
+    root_paths = [
+        "vsix_sandbox"
     ],
+    visibility = ["//integration:__subpackages__"],
 )

WORKSPACE

@@ -2,9 +2,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "aspect_rules_js",
-    sha256 = "538049993bec3ee1ae9b1c3cd669156bca04eb67027b222883e47b0a2aed2e67",
-    strip_prefix = "rules_js-1.0.0",
-    url = "https://github.com/aspect-build/rules_js/archive/refs/tags/v1.0.0.tar.gz",
+    sha256 = "b9fde0f20de6324ad443500ae738bda00facbd73900a12b417ce794856e01407",
+    strip_prefix = "rules_js-1.5.0",
+    url = "https://github.com/aspect-build/rules_js/archive/refs/tags/v1.5.0.tar.gz",
 )
 
 load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies")
@@ -13,9 +13,9 @@ rules_js_dependencies()
 
 http_archive(
     name = "aspect_rules_ts",
-    sha256 = "1945d5a356d0ec85359dea411467dec0f98502503a53798ead7f54aef849598b",
-    strip_prefix = "rules_ts-1.0.0-rc1",
-    url = "https://github.com/aspect-build/rules_ts/archive/refs/tags/v1.0.0-rc1.tar.gz",
+    sha256 = "743f0e988e4e3f1e25e52c79f9dc3da1ddd77507ae88787ae95b4e70c537872b",
+    strip_prefix = "rules_ts-1.0.0-rc4",
+    url = "https://github.com/aspect-build/rules_ts/archive/refs/tags/v1.0.0-rc4.tar.gz",
 )
 
 load("@aspect_rules_ts//ts:repositories.bzl", "rules_ts_dependencies")
@@ -24,9 +24,9 @@ rules_ts_dependencies(ts_version_from = "//:package.json",)
 
 http_archive(
     name = "aspect_rules_jasmine",
-    sha256 = "741d3376fdbf0c0c742e3bac0f854b1d49dbe1998d3530ef6f22f467675ca177",
-    strip_prefix = "rules_jasmine-0.0.1",
-    url = "https://github.com/aspect-build/rules_jasmine/archive/refs/tags/v0.0.1.tar.gz",
+    sha256 = "938a2818100fd89e7600a45b7ba4fcd4114c11c5b5741db30ff7c6e0dcb2ea4b",
+    strip_prefix = "rules_jasmine-0.1.0",
+    url = "https://github.com/aspect-build/rules_jasmine/archive/refs/tags/v0.1.0.tar.gz",
 )
 
 load("@aspect_rules_jasmine//jasmine:dependencies.bzl", "rules_jasmine_dependencies")
@@ -35,9 +35,9 @@ rules_jasmine_dependencies()
 
 http_archive(
     name = "aspect_rules_esbuild",
-    sha256 = "6446a784e72b04c33bc48debd84c5a54db4727f0a4a61a0da9faa64bededd7c2",
-    strip_prefix = "rules_esbuild-0909898c1344569f59fa83f70a1db7548563c274",
-    url = "https://github.com/aspect-build/rules_esbuild/archive/0909898c1344569f59fa83f70a1db7548563c274.tar.gz",
+    sha256 = "1e365451341ffb2490193292dfd9953f2ca009586c2381cb4dc08d01e48866b7",
+    strip_prefix = "rules_esbuild-0.12.0",
+    url = "https://github.com/aspect-build/rules_esbuild/archive/refs/tags/v0.12.0.tar.gz",
 )
 
 load("@aspect_rules_esbuild//esbuild:dependencies.bzl", "rules_esbuild_dependencies")
@@ -58,11 +58,11 @@ esbuild_register_toolchains(
     esbuild_version = ESBUILD_LATEST_VERSION,
 )
 
-load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains")
+load("@rules_nodejs//nodejs:repositories.bzl", "nodejs_register_toolchains")
 
 nodejs_register_toolchains(
     name = "nodejs",
-    node_version = DEFAULT_NODE_VERSION,
+    node_version = "14.20.0",
 )
 
 load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock")
@@ -71,6 +71,7 @@ npm_translate_lock(
     name = "npm",
     yarn_lock = "//:yarn.lock",
     package_json = "//:package.json",
+    npmrc = "//:.npmrc",
     verify_node_modules_ignored = "//:.bazelignore",
     public_hoist_packages = {
         # Hoist transitive closure of npm deps needed for vsce; this set was determined manually by
@@ -96,6 +97,7 @@ npm_translate_lock(
     name = "npm_integration_workspace",
     yarn_lock = "//integration/workspace:yarn.lock",
     package_json = "//integration/workspace:package.json",
+    npmrc = "//:.npmrc",
     verify_node_modules_ignored = "//:.bazelignore",
 )
 
@@ -107,6 +109,7 @@ npm_translate_lock(
     name = "npm_integration_pre_apf_project",
     yarn_lock = "//integration/pre_apf_project:yarn.lock",
     package_json = "//integration/pre_apf_project:package.json",
+    npmrc = "//:.npmrc",
     verify_node_modules_ignored = "//:.bazelignore",
 )
 
@@ -118,6 +121,7 @@ npm_translate_lock(
     name = "npm_integration_project",
     yarn_lock = "//integration/project:yarn.lock",
     package_json = "//integration/project:package.json",
+    npmrc = "//:.npmrc",
     verify_node_modules_ignored = "//:.bazelignore",
 )
 

integration/e2e/BUILD.bazel

@@ -26,7 +26,6 @@ js_test(
         "//:npm",
         "//integration",
         "//integration/project",
-        "//server:index",
         # Depend on //:node_modules/jasmine as a temporary work-around for jasmine escaping its
         # runfiles likely due to esm import issue in rules_js https://github.com/aspect-build/rules_js/issues/362
         "//:node_modules/jasmine",

integration/e2e/completion_spec.ts

@@ -1,4 +1,3 @@
-
 import * as vscode from 'vscode';
 
 import {activate, COMPLETION_COMMAND, FOO_TEMPLATE_URI} from './helper';

integration/e2e/jasmine.ts

@@ -33,6 +33,8 @@ export async function run(): Promise<void> {
 
   console.log(`Expecting to run ${jasmine.specFiles.length} specs.`);
 
+  console.log(JSON.stringify(jasmine.specFiles, null, 2))
+
   if (jasmine.specFiles.length === 0) {
     throw new Error('No specs found');
   }

package.json

@@ -209,15 +209,19 @@
     "format": "scripts/format.sh",
     "watch": "tsc -b -w",
     "package": "scripts/build.sh package.json",
-    "test": "yarn compile:test && jasmine --config=jasmine.json",
-    "test:inspect": "yarn compile:test && node --inspect-brk node_modules/jasmine/bin/jasmine.js --config=jasmine.json",
-    "test:lsp": "yarn compile:integration && jasmine --config=integration/lsp/jasmine.json",
-    "test:lsp-bazel": "yarn bazel test --test_output=streamed //integration:lsp_test",
-    "test:e2e": "yarn compile:integration && node dist/integration/e2e",
-    "test:e2e-bazel": "yarn bazel test --test_output=streamed //integration/e2e:e2e_test",
-    "test:syntaxes": "yarn compile:syntaxes-test && yarn build:syntaxes && jasmine dist/syntaxes/test/driver.js",
-    "test:bazel": "yarn bazel test --test_tag_filters=unit_test //...",
-    "test:bazel-watch": "yarn ibazel test --test_tag_filters=unit_test //..."
+    "test": "yarn bazel test --test_tag_filters=unit_test //...",
+    "test:watch": "yarn ibazel test --test_tag_filters=unit_test //...",
+    "test:lsp": "yarn bazel test --test_output=streamed //integration/lsp:test",
+    "test:e2e": "yarn bazel test --test_output=streamed //integration/e2e:test",
+    "test:inspect-client": "yarn bazel run --config=debug //client/src/tests:test",
+    "test:inspect-common": "yarn bazel run --config=debug //common/tests:test",
+    "test:inspect-server": "yarn bazel run --config=debug //server/src/tests:test",
+    "test:inspect-syntaxes": "yarn bazel run --config=debug //syntaxes/test:test",
+    "test:legacy-inspect": "yarn compile:test && node --inspect-brk node_modules/jasmine/bin/jasmine.js --config=jasmine.json",
+    "test:legacy": "yarn compile:test && jasmine --config=jasmine.json",
+    "test:legacy-lsp": "yarn compile:integration && jasmine --config=integration/lsp/jasmine.json",
+    "test:legacy-e2e": "yarn compile:integration && node dist/integration/e2e",
+    "test:legacy-syntaxes": "yarn compile:syntaxes-test && yarn build:syntaxes && jasmine dist/syntaxes/test/driver.js"
   },
   "dependencies": {
     "@angular/language-service": "14.2.0",

scripts/build.sh

@@ -30,16 +30,32 @@ fi
 
 set -ex -o pipefail
 
+# Clean up from last build
+rm -rf dist
+
+# Run legacy yarn install to update the lock file since we may have modified the package.json above
+# and so that we can call bazel via yarn bazel
+yarn install
+
+# Build the npm package with bazel
+yarn bazel build //:npm
+
+# Copy the bazel built package to dist/npm_bazel
+# TODO: copy to dist/npm when ready to cut-over
+mkdir -p dist/npm_bazel
+cp -r bazel-bin/npm/ dist/npm_bazel
+chmod -R +w dist/npm_bazel
+
+################################################################################
+#### LEGACY PRE-BAZEL BUILD --- can be removed after cut-over
+################################################################################
+
 # Enable extended pattern matching features
 shopt -s extglob
 
 # Clean up from last build
-rm -rf dist
 rm -rf **/*.tsbuildinfo
 
-
-yarn install;
-
 # Build the client and server
 yarn run compile
 

scripts/test.sh

@@ -2,30 +2,8 @@
 
 set -ex -o pipefail
 
-# Install test project dependencies
-if [[ -z "${CI}" ]]; then
-  (cd integration/project && yarn)
-  (cd integration/pre_apf_project && yarn)
-  (cd integration/workspace && yarn)
-fi
-
-# Server unit tests
-yarn run test
-
-# Smoke test for LSP
-yarn run test:lsp
+yarn bazel test //...
 
 # E2E test that brings up full vscode
 # TODO: Disabled for now because it cannot be run on CircleCI
-# yarn run test:e2e
-
-# Syntaxes tests
-yarn run test:syntaxes
-# Check git status to see if there are modified files
-STATUS="$(git status --porcelain)"
-echo $STATUS
-# If the status contains modified files in the syntaxes folder, they are out of date
-if [[ "$STATUS" == *"syntaxes"* ]]; then
-  echo 'Syntax files are out-of-sync with source. Please run "yarn run build:syntaxes".'
-  exit 1
-fi
+# bazel test --test_output=streamed //integration/e2e:test

syntaxes/BUILD.bazel

@@ -1,5 +1,30 @@
-load("@aspect_rules_js//js:defs.bzl", "js_library")
+load("@aspect_rules_js//js:defs.bzl", "js_run_binary", "js_library")
 load("@aspect_rules_ts//ts:defs.bzl", "ts_config")
+load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files")
+
+js_run_binary(
+    name = "build",
+    tool = "//syntaxes/src:build",
+    outs = [
+        # bazel output files names in the output tree are prefixed with `_` so
+        # we can refer to both the source files & output files by label in the
+        # write_source_files target below.
+        "_expression.json",
+        "_inline-styles.json",
+        "_inline-template.json",
+        "_template.json",
+    ]
+)
+
+write_source_files(
+    name = "syntaxes",
+    files = {
+        "expression.json": "_expression.json",
+        "inline-styles.json": "_inline-styles.json",
+        "inline-template.json": "_inline-template.json",
+        "template.json": "_template.json",
+    }
+)
 
 js_library(
     name = "json",