WIP

ds

Author: devversion

.circleci/config.yml

@@ -312,17 +312,35 @@ jobs:
     resource_class: xlarge
     environment:
       SAUCE_USERNAME: "angular-components"
-      SAUCE_ACCESS_KEY: "63348201a846-eeb9-3ee4-300f-ea990b8a"
-      # Note: This number should not be too high because otherwise we might run into
-      # a rate limit exception.
-      KARMA_PARALLEL_BROWSERS: 2
+      SAUCE_ACCESS_KEY: "a8b099ae-f003-4ee3-9bee-648a10284336"
+      SAUCE_TUNNEL_IDENTIFIER: angular-material-<< pipeline.id >>
     steps:
     - checkout_and_rebase
     - *restore_cache
     - *setup_bazel_ci_config
     - *yarn_install
 
-    - run: ./scripts/circleci/run-saucelabs-tests.sh
+    - run:
+        name: Saucelabs tunnel
+        command: ./scripts/saucelabs/start-tunnel.sh
+        background: true
+    - run:
+        name: Waiting for Saucelabs tunnel
+        command: ./scripts/saucelabs/wait-tunnel.sh
+    - run:
+        name: Saucelabs browser daemon
+        command: yarn -s sauce-bazel-daemon
+        background: true
+
+    - run:
+        name: Running tests
+        command: |
+          TESTS=$(yarn -s bazel query --output label 'attr("tags", "saucelabs", //src/...)')
+          yarn -s bazel test ${TESTS} --jobs=2
+
+    - run:
+        name: Shutting down tunnel
+        command: ./scripts/saucelabs/stop-tunnel.sh
     - *slack_notify_on_failure
 
   # ----------------------------------

BUILD.bazel

@@ -26,11 +26,3 @@ genrule(
     outs = ["entry_points_manifest.json"],
     cmd = "echo '%s' > $@" % entryPoints,
 )
-
-genrule(
-    name = "saucelabs_tunnel_identifier",
-    outs = ["saucelabs_tunnel_identifier.txt"],
-    cmd = "$(execpath //scripts/saucelabs:extract-tunnel-identifier) > $@",
-    stamp = True,
-    tools = ["//scripts/saucelabs:extract-tunnel-identifier"],
-)

package.json

@@ -48,7 +48,8 @@
     "integration-tests:size-test": "bazel test //integration/size-test/...",
     "check-mdc-tests": "ts-node --project scripts/tsconfig.json scripts/check-mdc-tests.ts",
     "check-mdc-exports": "ts-node --project scripts/tsconfig.json scripts/check-mdc-exports.ts",
-    "check-tools": "yarn tsc --project tools/tsconfig-ci.json"
+    "check-tools": "yarn tsc --project tools/tsconfig-ci.json",
+    "sauce-bazel-daemon": "ts-node --project tools/saucelabs-bazel/tsconfig.json tools/saucelabs-bazel/background-service/cli.ts"
   },
   "version": "12.1.0-next.0",
   "dependencies": {

scripts/circleci/run-saucelabs-tests.sh

@@ -1,11 +1,4 @@
 {
-  "ChromeHeadlessCI": {
-    "base": "ChromeHeadless",
-    "flags": [
-      "--window-size=1024,768",
-      "--no-sandbox"
-    ]
-  },
   "ChromeLocalDebug": {
     "base": "Chrome",
     "flags": [

scripts/saucelabs/BUILD.bazel

@@ -1,39 +1,18 @@
-const {readFileSync} = require('fs');
 const {customLaunchers, platformMap} = require('./browser-providers');
 
 module.exports = karmaConfig => {
   const config = {
     plugins: [
-      require('karma-sauce-launcher')
+      require('angular_material/tools/saucelabs-bazel/launcher'),
     ],
     customLaunchers: customLaunchers,
-    sauceLabs: {
-      testName: 'Angular Material Unit Tests',
-      startConnect: false,
-      recordVideo: false,
-      recordScreenshots: false,
-      idleTimeout: 600,
-      commandTimeout: 600,
-      maxDuration: 5400,
-    },
-    browserNoActivityTimeout: 300000,
-    browserDisconnectTimeout: 180000,
-    browserDisconnectTolerance: 3,
-    captureTimeout: 180000,
+    browserNoActivityTimeout: 90000,
+    browserDisconnectTimeout: 90000,
+    browserDisconnectTolerance: 2,
+    captureTimeout: 90000,
     browsers: platformMap.saucelabs,
-
-    // Try Websocket for a faster transmission first. Fallback to polling if necessary.
-    transports: ['websocket', 'polling'],
+    transports: ['polling', 'websocket'],
   };
 
-  // The tunnel identifier has been written to a stamped genrule that access the tunnel
-  // identifier through a volatile status key. This ensures that test and build results
-  // are not invalidated whenever the tunnel identifier changes (which it does for each PR).
-  const tunnelIdentifier = readFileSync(
-      require.resolve('../saucelabs_tunnel_identifier.txt'), 'utf8');
-
-  config.sauceLabs.build = tunnelIdentifier;
-  config.sauceLabs.tunnelIdentifier = tunnelIdentifier;
-
   karmaConfig.set(config);
 };

scripts/saucelabs/extract-tunnel-identifier.sh

@@ -156,22 +156,25 @@ def ng_e2e_test_library(deps = [], tsconfig = None, **kwargs):
     )
 
 def karma_web_test(name, **kwargs):
-  tags = kwargs.pop("tags", [])
-  # Custom standalone web test that can be run to test against any browser
-  # that is manually connected to.
-  _karma_web_test(
-      name = "%s_bin" % name,
-      tags = ["manual"],
-      **kwargs
-  )
-
-  # Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1429
-  native.sh_test(
-      name = "%s" % name,
-      srcs = ["%s_bin" % name],
-      tags = tags + ["ibazel_notify_changes"],
-  )
+    tags = kwargs.pop("tags", [])
+    timeout = kwargs.pop("timeout", None)
 
+    # Custom standalone web test that can be run to test against any browser
+    # that is manually connected to.
+    _karma_web_test(
+        name = "%s_bin" % name,
+        tags = ["manual"],
+        timeout = timeout,
+        **kwargs
+    )
+
+    # Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1429
+    native.sh_test(
+        name = "%s" % name,
+        srcs = ["%s_bin" % name],
+        timeout = timeout,
+        tags = tags + ["ibazel_notify_changes"],
+    )
 
 def karma_web_test_suite(name, **kwargs):
     web_test_args = {}
@@ -205,12 +208,11 @@ def karma_web_test_suite(name, **kwargs):
 
     sauce_web_test_args = dict(**web_test_args)
     sauce_web_test_args["data"] = sauce_web_test_args.get("data", []) + [
-      "//:saucelabs_tunnel_identifier",
-      "//test:browser-providers.js",
-      "//test:karma-browsers.json"
+        "//test:browser-providers.js",
+        "//test:karma-browsers.json",
     ]
     sauce_web_test_args["deps"] = sauce_web_test_args.get("deps", []) + [
-      "@npm//karma-sauce-launcher"
+        "//tools/saucelabs-bazel/launcher",
     ]
 
     # Add a saucelabs target for these karma tests
@@ -219,6 +221,7 @@ def karma_web_test_suite(name, **kwargs):
         timeout = "long",
         config_file = "//test:karma-saucelabs.conf.js",
         tags = ["manual", "saucelabs", "no-remote-exec"],
+        flaky = False,
         **sauce_web_test_args
     )
 

test/karma-browsers.json

@@ -0,0 +1,12 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "saucelabs-bazel",
+    srcs = [
+        "browser.ts",
+        "ipc-messages.ts",
+    ],
+    deps = [],
+)

test/karma-saucelabs.conf.js

@@ -0,0 +1,14 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "background-service",
+    srcs = glob(["**/*.ts"]),
+    deps = [
+        "//tools/saucelabs-bazel",
+        "@npm//@types/node",
+        "@npm//@types/selenium-webdriver",
+        "@npm//chalk",
+    ],
+)

tools/defaults.bzl

@@ -0,0 +1,35 @@
+import {Browser} from '../browser';
+import {SaucelabsDaemon} from './saucelabs-daemon';
+
+//process.env.SAUCE_USERNAME = 'angular-components';
+//process.env.SAUCE_ACCESS_KEY = 'a8b099ae-f003-4ee3-9bee-648a10284336';
+//process.env.SAUCE_TUNNEL_IDENTIFIER = 'angular-material-d35114de-1499-47a4-8179-643a25ea97c6';
+
+const parallelExecutions = 2;
+const {platformMap, customLaunchers} = require('../../../test/browser-providers');
+
+const username = process.env.SAUCE_USERNAME;
+const accessKey = process.env.SAUCE_ACCESS_KEY;
+const tunnelIdentifier = process.env.SAUCE_TUNNEL_IDENTIFIER;
+
+if (!username || !accessKey) {
+  throw Error('Please set the `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` variables.');
+}
+
+if (!tunnelIdentifier) {
+  throw Error('No tunnel set up. Please set the `SAUCE_TUNNEL_IDENTIFIER` variable.');
+}
+
+const saucelabsBrowsers: Browser[] = platformMap.saucelabs
+  .map((n: string) => customLaunchers[n]);
+const browserInstances: Browser[] = [];
+
+for (let i = 0; i < parallelExecutions; i++) {
+  browserInstances.push(...saucelabsBrowsers);
+}
+
+console.error(browserInstances.map(d => d.browserName));
+
+// Start the daemon and launch the given browser.
+new SaucelabsDaemon(username, accessKey, {tunnelIdentifier})
+  .launchBrowsers(browserInstances);

tools/saucelabs-bazel/BUILD.bazel

@@ -0,0 +1,42 @@
+import {createServer, Server, Socket} from 'net';
+import {BackgroundServiceReceiveMessages, NoAvailableBrowserMessage} from '../ipc-messages';
+import {SaucelabsDaemon} from './saucelabs-daemon';
+
+let nextSocketId = 0;
+
+export class IpcServer {
+  private readonly _server: Server;
+  private _connections = new Map<number, Socket>();
+
+  constructor(private _service: SaucelabsDaemon, port = 5324) {
+    this._server = createServer(this._connectionHandler.bind(this));
+    this._server.listen(port, () => console.info('Daemon IPC server listening.'));
+  }
+
+  private _connectionHandler(socket: Socket) {
+    const socketId = nextSocketId++;
+    this._connections.set(socketId, socket);
+    socket.on('data', b => this._processMessage(socket, socketId, JSON.parse(b.toString())));
+  }
+
+  private _processMessage(
+      socket: Socket, socketId: number, message: BackgroundServiceReceiveMessages) {
+    switch (message.type) {
+      case 'start-test':
+        console.debug(`Starting tests for ${socketId}`);
+        if (!this._service.startTest(
+                {testId: socketId, pageUrl: message.url, requestedBrowserId: message.browserId})) {
+          this._noAvailableBrowser(socket);
+        }
+        break;
+      case 'end-test':
+        console.debug(`Ending tests for ${socketId}`);
+        this._service.endTest(socketId);
+        break;
+    }
+  }
+
+  private _noAvailableBrowser(socket: Socket) {
+    socket.write(JSON.stringify(new NoAvailableBrowserMessage()));
+  }
+}