ci: run bazel integration tests

Author: filipesilva

.circleci/config.yml

@@ -58,6 +58,18 @@ executors:
       NPM_CONFIG_PREFIX: ~/.npm-global
     resource_class: large
 
+  heavy-test-executor:
+    parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion
+    docker:
+      - image: circleci/node:<< parameters.nodeversion >>-browsers
+    working_directory: ~/ng
+    environment:
+      NPM_CONFIG_PREFIX: ~/.npm-global
+    resource_class: 2xlarge+
+
   windows-executor:
     # Same as https://circleci.com/orbs/registry/orb/circleci/windows, but named.
     working_directory: ~/ng
@@ -134,7 +146,9 @@ jobs:
           keys:
             - *cache_key
             - *cache_key_fallback
-      - run: yarn install --frozen-lockfile
+      # TODO: shouldn't have to disable frozen lockfile but CI seem to be bothered by yet and I want to test right now
+      # - run: yarn install --frozen-lockfile
+      - run: yarn install
       - persist_to_workspace:
           root: *workspace_location
           paths:
@@ -167,20 +181,13 @@ jobs:
       - custom_attach_workspace
       - run: yarn test --full
 
-  test-large:
-    parameters:
-      ve:
-        type: boolean
-        default: false
-      glob:
-        type: string
-        default: ""
-    executor: test-executor
-    parallelism: 4
+  integration:
+    executor: heavy-test-executor
     steps:
       - custom_attach_workspace
-      - run: yarn webdriver-update
-      - run: yarn test-large --full <<# parameters.ve >>--ve<</ parameters.ve >> <<# parameters.glob >>--glob="<< parameters.glob >>"<</ parameters.glob >> --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX}
+      - run: 
+          command: yarn bazel:integration
+          no_output_timeout: 60m
 
   e2e-cli:
     parameters:
@@ -247,17 +254,6 @@ jobs:
       - custom_attach_workspace
       - run: yarn build
 
-  # This is where we put all the misbehaving and flaky tests so we can fine-tune their conditions 
-  # and rerun them faster.
-  flake-jail:
-    executor: test-executor
-    resource_class: medium
-    steps:
-      - custom_attach_workspace
-      - run: yarn webdriver-update
-      - run: yarn test-large --full --flakey --ve
-      - run: yarn test-large --full --flakey
-
   build-bazel:
     executor: action-executor
     resource_class: large
@@ -365,13 +361,8 @@ workflows:
       - test:
           requires:
             - build
-      - test-large:
-          requires:
-            - build
-      - test-large:
-          name: test-large-ve
-          ve: true
-          glob: "packages/angular_devkit/@(build_angular|build_ng_packagr)/@(src|test)/@(build|browser)/*_spec_large.ts"
+      # TODO: ng-packagr spec large also needs to be moved to bazel
+      - integration:
           requires:
             - build
       - e2e-cli:
@@ -414,9 +405,6 @@ workflows:
       - test-browsers:
           requires:
             - build
-      - flake-jail:
-          requires:
-            - build
 
       # Windows jobs
       # These jobs only run after their non-windows counterparts finish successfully.

integration/build_angular/BUILD.bazel

@@ -68,6 +68,7 @@ INTEGRATION_TESTS = [
             "//packages/angular_devkit/core:npm_package_archive": "@angular-devkit/core",
             "//packages/ngtools/webpack:npm_package_archive": "@ngtools/webpack",
         },
+        # TODO: figure out if CI can handle this not being exclusive.
         tags = ["exclusive"],
         test_files = ":build_angular_sources",
     )

package.json

@@ -23,7 +23,8 @@
     "bazel:format": "find . -type f \\( -name \"*.bzl\" -or -name WORKSPACE -or -name BUILD -or -name BUILD.bazel \\) ! -path \"*/node_modules/*\" | xargs buildifier -v --warnings=attr-cfg,attr-license,attr-non-empty,attr-output-default,attr-single-file,constant-glob,ctx-args,depset-iteration,depset-union,dict-concatenation,duplicated-name,filetype,git-repository,http-archive,integer-division,load,load-on-top,native-build,native-package,output-group,package-name,package-on-top,positional-args,redefined-variable,repository-name,same-origin-load,string-iteration,unused-variable",
     "bazel:lint": "yarn bazel:format --lint=warn",
     "bazel:lint-fix": "yarn bazel:format --lint=fix",
-    "bazel:test": "bazel test //...",
+    "bazel:test": "bazel test //packages/... //etc/... ",
+    "bazel:integration": "bazel test //integration/...",
     "build": "node ./bin/devkit-admin build",
     "build-tsc": "tsc -p tsconfig.json",
     "debug:test": "node --inspect-brk ./bin/devkit-admin test",

scripts/test.ts

@@ -19,13 +19,6 @@ import { packages } from '../lib/packages';
 
 const Jasmine = require('jasmine');
 
-const knownFlakes = [
-  // Rebuild tests in test-large are flakey if not run as the first suite.
-  // https://github.com/angular/angular-cli/pull/15204
-  'packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts',
-  'packages/angular_devkit/build_angular/test/browser/web-worker_spec_large.ts',
-];
-
 const projectBaseDir = join(__dirname, '..');
 require('source-map-support').install({
   hookRequire: true,
@@ -194,9 +187,6 @@ export default function(args: ParsedArgs, logger: logging.Logger) {
     }
   }
 
-  // Filter in/out flakes according to the --flakey flag.
-  tests = tests.filter(test => !!args.flakey == knownFlakes.includes(test.replace(/[\/\\]/g, '/')));
-
   if (args.shard !== undefined) {
     // Remove tests that are not part of this shard.
     const shardId = args['shard'];