Separate Saucelabs cross-browser tests by package (#1756)

Set up Saucelabs tests to run separately, on CI or locally.

Author: hsubox76

.travis.yml

@@ -42,20 +42,15 @@ matrix:
 jobs:
   allow_failures:
     - script: yarn test:saucelabs
-    - script: yarn test:saucelabs --database-firestore-only
   include:
     - name: Node.js and Browser (Chrome) Test
       stage: test
       script: xvfb-run yarn test
       after_success: yarn test:coverage
-    - name: Cross Browser Test for SDKs except Database and Firestore
+    - name: Cross Browser Test for SDKs
       stage: test
       script: yarn test:saucelabs
       if: type = push
-    - name: Cross Browser Test for Database and Firestore SDK
-      stage: test
-      script: yarn test:saucelabs --database-firestore-only
-      if: type = push
     - stage: deploy
       script: skip
       # NPM Canary Build Config

config/karma.saucelabs.js

@@ -16,42 +16,98 @@
  */
 
 const argv = require('yargs').argv;
-const glob = require('glob');
-const karma = require('karma');
 const path = require('path');
 const karmaBase = require('./karma.base');
 
+// karma.conf.js test configuration file to run.
+const testConfigFile = argv['testConfigFile'];
+if (!testConfigFile) {
+  console.error('No test file path provided.');
+  process.exit(1);
+}
+
+/**
+ * Custom SauceLabs Launchers
+ */
+const browserMap = {
+  // Desktop
+  Chrome_Windows: seleniumLauncher('chrome', 'Windows 10', 'latest'),
+  Firefox_Windows: seleniumLauncher('firefox', 'Windows 10', 'latest'),
+  Safari_macOS: seleniumLauncher('safari', 'macOS 10.13', 'latest'),
+  Edge_Windows: seleniumLauncher('MicrosoftEdge', 'Windows 10', 'latest'),
+  IE_Windows: seleniumLauncher('internet explorer', 'Windows 10', 'latest')
+
+  // Mobile
+  // Safari_iOS: appiumLauncher('Safari', 'iPhone Simulator', 'iOS', '11.2'),
+  // Chrome_Android: appiumLauncher('Chrome', 'Android Emulator', 'Android', '6.0')
+};
+
+/**
+ * Any special options per package.
+ */
+const packageConfigs = {
+  messaging: {
+    // Messaging currently only supports these browsers.
+    browsers: ['Chrome_Windows', 'Firefox_Windows', 'Edge_Windows']
+  }
+};
+
+/**
+ * Gets the browser/launcher map for this package.
+ *
+ * @param {string} packageName Name of package being tested (e.g., "firestore")
+ */
+function getSauceLabsBrowsers(packageName) {
+  if (packageConfigs[packageName]) {
+    const filteredBrowserMap = {};
+    for (const browserKey in browserMap) {
+      if (packageConfigs[packageName].browsers.includes(browserKey)) {
+        filteredBrowserMap[browserKey] = browserMap[browserKey];
+      }
+    }
+    return filteredBrowserMap;
+  } else {
+    return browserMap;
+  }
+}
+
+/**
+ * Get package name from package path command line arg.
+ */
+function getPackageLabels() {
+  const match = testConfigFile.match(
+    /([a-zA-Z]+)\/([a-zA-Z]+)\/karma\.conf\.js/
+  );
+  return {
+    type: match[1],
+    name: match[2]
+  };
+}
+
 /**
  * Gets a list of file patterns for test, defined individually
  * in karma.conf.js in each package under worksapce packages or
  * integration.
  */
 function getTestFiles() {
   let root = path.resolve(__dirname, '..');
-  configs = argv['database-firestore-only']
-    ? glob.sync('packages/{database,firestore}/karma.conf.js')
-    : glob.sync('{packages,integration}/*/karma.conf.js', {
-        // Excluded due to flakiness or long run time.
-        ignore: [
-          'packages/database/*',
-          'packages/firestore/*',
-          'integration/firestore/*',
-          'integration/messaging/*'
-        ]
-      });
-  files = configs.map(x => {
-    let patterns = require(path.join(root, x)).files;
-    let dirname = path.dirname(x);
-    return patterns.map(p => path.join(dirname, p));
-  });
-  return [].concat(...files);
+  const { name: packageName } = getPackageLabels();
+  let patterns = require(path.join(root, testConfigFile)).files;
+  let dirname = path.dirname(testConfigFile);
+  return { packageName, files: patterns.map(p => path.join(dirname, p)) };
 }
 
 function seleniumLauncher(browserName, platform, version) {
+  const { name, type } = getPackageLabels();
+  const testName =
+    type === 'integration'
+      ? `${type}-${name}-${browserName}`
+      : `${name}-${browserName}`;
   return {
     base: 'SauceLabs',
     browserName: browserName,
     extendedDebugging: 'true',
+    name: testName,
     recordLogs: 'true',
     recordVideo: 'true',
     recordScreenshots: 'true',
@@ -81,27 +137,32 @@ function appiumLauncher(
   };
 }
 
-/**
- * Custom SauceLabs Launchers
- */
-const sauceLabsBrowsers = {
-  // Desktop
-  Chrome_Windows: seleniumLauncher('chrome', 'Windows 10', 'latest'),
-  Firefox_Windows: seleniumLauncher('firefox', 'Windows 10', 'latest'),
-  Safari_macOS: seleniumLauncher('safari', 'macOS 10.13', 'latest'),
-  Edge_Windows: seleniumLauncher('MicrosoftEdge', 'Windows 10', 'latest'),
-  IE_Windows: seleniumLauncher('internet explorer', 'Windows 10', 'latest')
-
-  // Mobile
-  // Safari_iOS: appiumLauncher('Safari', 'iPhone Simulator', 'iOS', '11.2'),
-  // Chrome_Android: appiumLauncher('Chrome', 'Android Emulator', 'Android', '6.0')
-};
-
 module.exports = function(config) {
+  const { packageName, files: testFiles } = getTestFiles();
+  const sauceLabsBrowsers = getSauceLabsBrowsers(packageName);
+
+  const sauceLabsConfig = {
+    tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER + '-' + packageName,
+    build: process.env.TRAVIS_BUILD_NUMBER || argv['buildNumber'],
+    username: process.env.SAUCE_USERNAME,
+    accessKey: process.env.SAUCE_ACCESS_KEY,
+    startConnect: true,
+    connectOptions: {
+      // Realtime Database uses WebSockets to connect to firebaseio.com
+      // so we have to set noSslBumpDomains. Theoretically SSL Bumping
+      // only needs to be disabled for 'firebaseio.com'. However, we are
+      // seeing much longer test time with that configuration, so leave
+      // it as 'all' for now.
+      // See https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect
+      // for more details.
+      noSslBumpDomains: 'all'
+    }
+  };
+
   const karmaConfig = Object.assign({}, karmaBase, {
     basePath: '../',
 
-    files: ['packages/polyfill/index.ts', ...getTestFiles()],
+    files: ['packages/polyfill/index.ts', ...testFiles],
 
     logLevel: config.LOG_INFO,
 
@@ -151,22 +212,7 @@ module.exports = function(config) {
       overviewColumn: false
     },
 
-    sauceLabs: {
-      tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER,
-      username: process.env.SAUCE_USERNAME,
-      accessKey: process.env.SAUCE_ACCESS_KEY,
-      startConnect: true,
-      connectOptions: {
-        // Realtime Database uses WebSockets to connect to firebaseio.com
-        // so we have to set noSslBumpDomains. Theoretically SSL Bumping
-        // only needs to be disabled for 'firebaseio.com'. However, we are
-        // seeing much longer test time with that configuration, so leave
-        // it as 'all' for now.
-        // See https://wiki.saucelabs.com/display/DOCS/Troubleshooting+Sauce+Connect
-        // for more details.
-        noSslBumpDomains: 'all'
-      }
-    }
+    sauceLabs: sauceLabsConfig
   });
 
   config.set(karmaConfig);

config/webpack.test.js

@@ -30,7 +30,8 @@ module.exports = {
           loader: 'ts-loader',
           options: {
             compilerOptions: {
-              module: 'commonjs'
+              module: 'commonjs',
+              downlevelIteration: true
             }
           }
         }

package.json

@@ -29,8 +29,8 @@
     "pretest:coverage": "mkdirp coverage",
     "test:coverage": "lcov-result-merger 'packages/**/lcov.info' | coveralls",
     "test:setup": "node tools/config.js",
-    "pretest:saucelabs": "lerna run --parallel pretest",
-    "test:saucelabs": "karma start config/karma.saucelabs.js --single-run",
+    "test:saucelabs": "node scripts/run_saucelabs.js",
+    "test:saucelabs:single": "karma start config/karma.saucelabs.js --single-run",
     "docgen:js": "node scripts/docgen/generate-docs.js --api js",
     "docgen:node": "node scripts/docgen/generate-docs.js --api node",
     "docgen": "yarn docgen:js; yarn docgen:node",

packages/installations/karma.conf.js

@@ -17,11 +17,15 @@
 
 const karmaBase = require('../../config/karma.base');
 
+const files = ['src/**/*.test.ts'];
+
 module.exports = function(config) {
   config.set({
     ...karmaBase,
-    files: ['src/**/*.test.ts'],
+    files,
     preprocessors: { '**/*.ts': ['webpack', 'sourcemap'] },
     frameworks: ['mocha']
   });
 };
+
+module.exports.files = files;

packages/performance/karma.conf.js

@@ -16,14 +16,18 @@
 
 const karmaBase = require('../../config/karma.base');
 
+const files = [`test/**/*`, 'src/**/*.test.ts'];
+
 module.exports = function(config) {
   config.set({
     ...karmaBase,
     // files to load into karma
-    files: [`test/**/*`, 'src/**/*.test.ts'],
+    files,
     preprocessors: { '**/*.ts': ['webpack', 'sourcemap'] },
     // frameworks to use
     // available frameworks: https://npmjs.org/browse/keyword/karma-adapter
     frameworks: ['mocha']
   });
 };
+
+module.exports.files = files;

scripts/run_saucelabs.js

@@ -0,0 +1,127 @@
+/**
+ * @license
+ * Copyright 2019 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { spawn } = require('child-process-promise');
+const { exists } = require('mz/fs');
+const yargs = require('yargs');
+const glob = require('glob');
+const path = require('path');
+
+// Check for 'configFiles' flag to run on specified karma.conf.js files instead
+// of on all files.
+const { configFiles } = yargs
+  .option('configFiles', {
+    default: [],
+    describe: 'specify individual karma.conf.js files to run on',
+    type: 'array'
+  })
+  .version(false)
+  .help().argv;
+
+// Get all karma.conf.js files that need to be run.
+// runNextTest() pulls filenames one-by-one from this queue.
+const testFiles = configFiles.length
+  ? configFiles
+  : glob
+      .sync(`{packages,integration}/*/karma.conf.js`)
+      // Automated tests in integration/firestore are currently disabled.
+      .filter(name => !name.includes('integration/firestore'));
+
+// Get CI build number or generate one if running locally.
+const buildNumber =
+  process.env.TRAVIS_BUILD_NUMBER ||
+  `local_${process.env.USER}_${new Date().getTime()}`;
+
+/**
+ * Runs a set of SauceLabs browser tests based on this karma config file.
+ *
+ * @param {string} testFile Path to karma.conf.js file that defines this test
+ * group.
+ */
+async function runTest(testFile) {
+  // Run pretest if this dir has a package.json with a pretest script.
+  const testFileDir =
+    path.resolve(__dirname, '../') + '/' + path.dirname(testFile);
+  const pkgPath = testFileDir + '/package.json';
+  if (await exists(pkgPath)) {
+    const pkg = require(pkgPath);
+    if (pkg.scripts.pretest) {
+      await spawn('yarn', ['--cwd', testFileDir, 'pretest'], {
+        stdio: 'inherit'
+      });
+    }
+  }
+
+  const promise = spawn('yarn', [
+    'test:saucelabs:single',
+    '--testConfigFile',
+    testFile,
+    '--buildNumber',
+    buildNumber
+  ]);
+  const childProcess = promise.childProcess;
+  let exitCode = 0;
+
+  childProcess.stdout.on('data', data => {
+    console.log(`[${testFile}]:`, data.toString());
+  });
+
+  // Lerna's normal output goes to stderr for some reason.
+  childProcess.stderr.on('data', data => {
+    console.log(`[${testFile}]:`, data.toString());
+  });
+
+  // Capture exit code of this single package test run
+  childProcess.on('exit', code => {
+    exitCode = code;
+  });
+
+  return promise
+    .then(() => {
+      console.log(`[${testFile}] ******* DONE *******`);
+      return exitCode;
+    })
+    .catch(err => {
+      console.error(`[${testFile}] ERROR:`, err.message);
+      return exitCode;
+    });
+}
+
+/**
+ * Runs next file in testFiles queue as long as there are files in the queue.
+ *
+ * @param {number} maxExitCode Current highest exit code from all processes
+ * run so far. When main process is complete, it will exit with highest code
+ * of all child processes.  This allows any failing test to result in a CI
+ * build failure for the whole Saucelabs run.
+ */
+async function runNextTest(maxExitCode = 0) {
+  // When test queue is empty, exit with code 0 if no tests failed or
+  // 1 if any tests failed.
+  if (!testFiles.length) process.exit(maxExitCode);
+  const nextFile = testFiles.shift();
+  let exitCode;
+  try {
+    exitCode = await runTest(nextFile);
+  } catch (e) {
+    console.error(`[${nextFile}] ERROR:`, e.message);
+    exitCode = 1;
+  }
+  runNextTest(Math.max(exitCode, maxExitCode));
+}
+
+runNextTest();