build: run all unit tests on browserstack

* No longer runs tests on Saucelabs, because those jobs mostly time out, or are just flaky.

Author: devversion

.travis.yml

@@ -27,9 +27,9 @@ jobs:
     - env: "MODE=payload"
     - env: "MODE=prerender"
     - env: "MODE=e2e"
-    - env: "MODE=saucelabs_required"
-    - env: "MODE=browserstack_required"
-    - env: "MODE=travis_required"
+    - env: "MODE=test-browserstack-1"
+    - env: "MODE=test-browserstack-2"
+    - env: "MODE=test-travis-1"
     - env: "DEPLOY_MODE=build-artifacts"
       if: type = push
     - env: "DEPLOY_MODE=docs-content"
@@ -45,8 +45,8 @@ env:
   global:
   - LOGS_DIR=/tmp/angular-material2-build/logs
   - SAUCE_USERNAME=angular-ci
-  - BROWSER_STACK_USERNAME=angularteam1
-  - BROWSER_STACK_ACCESS_KEY=BWCd4SynLzdDcv8xtzsB
+  - BROWSER_STACK_USERNAME=paulgschwendtner2
+  - BROWSER_STACK_ACCESS_KEY=hsz2pirQRsDASY1aZKaz
   - BROWSER_PROVIDER_READY_FILE=/tmp/angular-material2-build/readyfile
   - BROWSER_PROVIDER_ERROR_FILE=/tmp/angular-material2-build/errorfile
 

package.json

@@ -100,7 +100,6 @@
     "karma-coverage": "^1.1.1",
     "karma-firefox-launcher": "^1.0.1",
     "karma-jasmine": "^1.1.0",
-    "karma-sauce-launcher": "^1.2.0",
     "karma-sourcemap-loader": "^0.3.7",
     "madge": "^2.2.0",
     "magic-string": "^0.22.4",

scripts/browserstack/start-tunnel.sh

@@ -30,14 +30,15 @@ unzip -q ${TUNNEL_FILE} -d browserstack-tunnel
 # Cleanup the download directory.
 rm ${TUNNEL_FILE}
 
-ARGS=""
+if [ -z "${TRAVIS_JOB_ID}" ]; then
+  echo "Error: Cannot set up a BrowserStack tunnel if there is no '\$TRAVIS_JOB_ID' set."
+  exit 1
+fi;
 
-# Set tunnel-id only on Travis, to make local testing easier.
-if [ ! -z "${TRAVIS_JOB_ID}" ]; then
-  ARGS="${ARGS} --local-identifier ${TRAVIS_JOB_ID}"
-fi
+ARGS="--local-identifier ${TRAVIS_JOB_ID}"
 
-echo "Starting Browserstack Local in the background, logging into: ${TUNNEL_LOG}"
+echo "Starting Browserstack Local in the background, logging into: ${TUNNEL_LOG}." \
+  "Using the following tunnel identifier: ${TRAVIS_JOB_ID}"
 
 # Extension to the BrowserStackLocal binaries, because those can't create a readyfile.
 function create_ready_file {

scripts/ci/sources/mode.sh

@@ -22,7 +22,7 @@ is_payload() {
 }
 
 is_unit() {
-  [[ "${MODE}" =~ ^.*_(optional|required)$ ]]
+  [[ "${MODE}" =~ ^test-.* ]]
 }
 
 is_prerender() {

scripts/ci/sources/tunnel.sh

@@ -6,38 +6,38 @@ source ./scripts/retry-call.sh
 # Variable the specifies how often the wait script should be invoked if it fails.
 WAIT_RETRIES=2
 
-start_tunnel() {
+start_tunnel_if_necessary() {
   case "$MODE" in
-    e2e*|saucelabs*)
+    e2e*)
       ./scripts/saucelabs/start-tunnel.sh
       ;;
-    browserstack*)
+    test-browserstack-*)
       ./scripts/browserstack/start-tunnel.sh
       ;;
     *)
       ;;
   esac
 }
 
-wait_for_tunnel() {
+wait_for_tunnel_if_present() {
   case "$MODE" in
-    e2e*|saucelabs*)
+    e2e*)
       retryCall ${WAIT_RETRIES} ./scripts/saucelabs/wait-tunnel.sh
       ;;
-    browserstack*)
+    test-browserstack-*)
       retryCall ${WAIT_RETRIES} ./scripts/browserstack/wait-tunnel.sh
       ;;
     *)
       ;;
   esac
 }
 
-teardown_tunnel() {
+teardown_tunnel_if_present() {
   case "$MODE" in
-    e2e*|saucelabs*)
+    e2e*)
       ./scripts/saucelabs/stop-tunnel.sh
       ;;
-    browserstack*)
+    test-browserstack-*)
       ./scripts/browserstack/stop-tunnel.sh
       ;;
     *)

scripts/ci/travis-testing.sh

@@ -28,8 +28,8 @@ if [ "$TRAVIS_PULL_REQUEST" = "true" ]; then
   fi
 fi
 
-start_tunnel
-wait_for_tunnel
+start_tunnel_if_necessary
+wait_for_tunnel_if_present
 
 if is_lint; then
   $(npm bin)/gulp ci:lint
@@ -52,4 +52,4 @@ if [ -f dist/coverage/coverage-summary.json ]; then
   $(npm bin)/gulp ci:coverage
 fi
 
-teardown_tunnel
+teardown_tunnel_if_present

test/browser-providers.js

@@ -5,75 +5,61 @@
  * Target can be either: BS (Browserstack) | SL (Saucelabs) | TC (Travis CI) | null (To not run)
  */
 const browserConfig = {
-  'ChromeHeadlessCI':  { unitTest: {target: 'TC', required: true  }},
-  'FirefoxHeadless':   { unitTest: {target: 'TC', required: true  }},
-  'ChromeBeta':        { unitTest: {target: null, required: false }},
-  'FirefoxBeta':       { unitTest: {target: null, required: false }},
-  'ChromeDev':         { unitTest: {target: null, required: true  }},
-  'FirefoxDev':        { unitTest: {target: null, required: true  }},
-  'IE9':               { unitTest: {target: null, required: false }},
-  'IE10':              { unitTest: {target: null, required: true  }},
-  'IE11':              { unitTest: {target: 'SL', required: true  }},
-  'Edge':              { unitTest: {target: 'SL', required: true  }},
-  'Android4.1':        { unitTest: {target: null, required: false }},
-  'Android4.2':        { unitTest: {target: null, required: false }},
-  'Android4.3':        { unitTest: {target: null, required: false }},
-  'Android4.4':        { unitTest: {target: null, required: false }},
-  'Android5':          { unitTest: {target: null, required: false }},
-  'Safari7':           { unitTest: {target: null, required: false }},
-  'Safari8':           { unitTest: {target: null, required: false }},
-  'Safari9':           { unitTest: {target: 'SL', required: true  }},
-  'Safari10':          { unitTest: {target: 'BS', required: true  }},
-  'iOS7':              { unitTest: {target: null, required: false }},
-  'iOS8':              { unitTest: {target: null, required: false }},
-  'iOS9':              { unitTest: {target: null, required: false }},
-  'iOS10':             { unitTest: {target: 'BS', required: true  }},
-  'WindowsPhone':      { unitTest: {target: null, required: false }}
+  'ChromeHeadlessCI':  { target: 'TC', poolId: 1 },
+  'FirefoxHeadless':   { target: 'TC', poolId: 1 },
+  'IE11':              { target: null },
+  'Edge':              { target: 'BS', poolId: 1 },
+  'Android4.4':        { target: null },
+  'Android5':          { target: null },
+  'Safari10':          { target: 'BS', poolId: 1 },
+  'Safari11':          { target: 'BS', poolId: 2 },
+  'iOS10':             { target: null },
+  'iOS11':             { target: 'BS', poolId: 2 },
+  'WindowsPhone':      { target: null }
 };
 
 /** Exports all available remote browsers. */
 exports.customLaunchers = require('./remote_browsers.json');
 
 /** Exports a map of configured browsers, which should run on the CI. */
 exports.platformMap = {
-  'saucelabs': {
-    required: buildConfiguration('unitTest', 'SL', true),
-    optional: buildConfiguration('unitTest', 'SL', false)
-  },
-  'browserstack': {
-    required: buildConfiguration('unitTest', 'BS', true),
-    optional: buildConfiguration('unitTest', 'BS', false)
-  },
-  'travis': {
-    required: buildConfiguration('unitTest', 'TC', true),
-    optional: buildConfiguration('unitTest', 'TC', false)
-  }
+  'browserstack': buildConfiguration('BS'),
+  'travis': buildConfiguration('TC'),
 };
 
-/** Build a list of configuration (custom launcher names). */
-function buildConfiguration(type, target, required) {
-  const targetBrowsers = Object.keys(browserConfig)
-    .map(browserName => [browserName, browserConfig[browserName][type]])
-    .filter(([, config]) => config.required === required && config.target === target)
-    .map(([browserName]) => browserName);
+/** Build a list of configuration for the specified platform. */
+function buildConfiguration(platform) {
+  const platformConfig = {};
 
-  // For browsers that run on Travis CI the browser name shouldn't be prefixed with the shortcut
-  // of Travis. The different Karma launchers only work with the plain browser name (e.g Firefox)
-  if (target === 'TC') {
-    return targetBrowsers;
-  }
+  Object.keys(browserConfig).forEach(browserName => {
+    const config = browserConfig[browserName];
 
-  return targetBrowsers.map(browserName => `${target}_${browserName.toUpperCase()}`);
+    if (config.target !== platform || !config.poolId) {
+      return;
+    }
+
+    if (!platformConfig[config.poolId]) {
+      platformConfig[config.poolId] = [];
+    }
+
+    // For browsers that run on Travis CI the browser name shouldn't be prefixed with the shortcut
+    // of Travis. The different Karma launchers only work with the plain browser name (e.g Firefox)
+    if (platform !== 'TC') {
+      browserName = `${platform}_${browserName.toUpperCase()}`;
+    }
+
+    platformConfig[config.poolId].push(browserName);
+  });
+
+  return platformConfig;
 }
 
 /** Decode the token for Travis to use. */
 function decodeToken(token) {
   return (token || '').split('').reverse().join('');
 }
 
-
 /** Ensures that the Travis access keys work properly. */
 if (process.env.TRAVIS) {
-  process.env.SAUCE_ACCESS_KEY = decodeToken(process.env.SAUCE_ACCESS_KEY);
   process.env.BROWSER_STACK_ACCESS_KEY = decodeToken(process.env.BROWSER_STACK_ACCESS_KEY);
 }

test/karma.conf.js

@@ -9,7 +9,6 @@ module.exports = (config) => {
     plugins: [
       require('karma-jasmine'),
       require('karma-browserstack-launcher'),
-      require('karma-sauce-launcher'),
       require('karma-chrome-launcher'),
       require('karma-firefox-launcher'),
       require('karma-sourcemap-loader'),
@@ -79,9 +78,10 @@ module.exports = (config) => {
 
     browserDisconnectTimeout: 20000,
     browserNoActivityTimeout: 240000,
+    browserDisconnectTolerance: 1,
     captureTimeout: 120000,
-    browsers: ['ChromeHeadlessLocal'],
 
+    browsers: ['ChromeHeadlessLocal'],
     singleRun: false,
 
     browserConsoleLogOptions: {
@@ -98,31 +98,30 @@ module.exports = (config) => {
   });
 
   if (process.env['TRAVIS']) {
-    const buildId = `TRAVIS #${process.env.TRAVIS_BUILD_NUMBER} (${process.env.TRAVIS_BUILD_ID})`;
+    const buildId = `TravisCI ${process.env.TRAVIS_BUILD_ID}`;
+    const tunnelIdentifier = process.env.TRAVIS_JOB_ID;
 
-    if (process.env['TRAVIS_PULL_REQUEST'] === 'false' &&
-        process.env['MODE'] === "travis_required") {
+    if (process.env['TRAVIS_PULL_REQUEST'] === 'false'
+        && process.env['MODE'] === "test-travis-1") {
 
       config.preprocessors['dist/packages/**/!(*+(.|-)spec).js'] = ['coverage'];
       config.reporters.push('coverage');
     }
 
     // The MODE variable is the indicator of what row in the test matrix we're running.
-    // It will look like <platform>_<target>, where platform is one of 'saucelabs', 'browserstack'
-    // or 'travis'. The target is a reference to different collections of browsers that can run
-    // in the previously specified platform.
-    const [platform, target] = process.env.MODE.split('_');
-
-    if (platform === 'saucelabs') {
-      config.sauceLabs.build = buildId;
-      config.sauceLabs.tunnelIdentifier = process.env.TRAVIS_JOB_ID;
-    } else if (platform === 'browserstack') {
+    // It will look like "test-<platform>[-poolId], where platform is one of 'browserstack' or
+    // 'travis'. Pool ids allow running different browsers on the same
+    // platform, but in different CI jobs.
+    const [, platform, poolId] = process.env.MODE.split('-');
+
+    if (platform === 'browserstack') {
       config.browserStack.build = buildId;
-      config.browserStack.tunnelIdentifier = process.env.TRAVIS_JOB_ID;
+      config.browserStack.tunnelIdentifier = tunnelIdentifier;
+      console.log(`Setting up Browserstack launcher. Connecting to tunnel: "${tunnelIdentifier}"`);
     } else if (platform !== 'travis') {
-      throw new Error(`Platform "${platform}" unknown, but Travis specified. Exiting.`);
+      throw new Error(`Platform "${platform}" is unknown. Exiting..`);
     }
 
-    config.browsers = platformMap[platform][target.toLowerCase()];
+    config.browsers = platformMap[platform][parseInt(poolId)];
   }
 };