test(e2e): initial setup for screenshot tests

Author: Robert Messerle

.gitattributes

@@ -0,0 +1,2 @@
+# Files ending with `.screenshot.png` will be stored using `git-lfs` rather than stored in the git repository
+*.screenshot.png filter=lfs diff=lfs merge=lfs -text

.travis.yml

@@ -5,7 +5,14 @@ language: node_js
 sudo: false
 
 node_js:
-  - '4.2.3'
+  - '5.6.0'
+
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - libstdc++6
 
 env:
   global:
@@ -33,6 +40,7 @@ env:
     - MODE=dart_required DART_CHANNEL=stable DART_VERSION=$DART_STABLE_VERSION
     - MODE=saucelabs_optional
     - MODE=browserstack_optional
+    - MODE=e2e
 
 matrix:
   allow_failures:

e2e/_screenshot.js

@@ -0,0 +1,81 @@
+"use strict";
+
+const path = require('path');
+const child_process  = require('child_process');
+const mapnik = require('mapnik');
+
+const SHA = process.env.TRAVIS_COMMIT;
+const LFS_BASE_URL = 'https://media.githubusercontent.com/media/angular/material2';
+
+/**
+ * Generates a screenshot from the current state of the Protractor test and compares it to the
+ * previous stored screenshot.  If the screenshots do not match or if no existing screenshot is
+ * found, an error will be thrown.  In both cases, the new screenshot will be stored so that it can
+ * be added to git.
+ */
+class Screenshot {
+  /**
+   * @param {string} id A unique identifier used for the screenshot
+   */
+  constructor(id) {
+    this.id   = id;
+    this.path = path.resolve(__dirname, '..', 'screenshots', id + '.screenshot.png');
+    this.url  = `${LFS_BASE_URL}/${SHA}/screenshots/${encodeURIComponent(id)}.screenshot.png`;
+    browser.takeScreenshot().then(png => this.storeScreenshot(png));
+  }
+
+  /**
+   * Stores a local copy of the screenshot for future comparison
+   * @param {string} png The base64-encoded screenshot generated from the current browser state
+   */
+  storeScreenshot(png) {
+    console.info(`[STATUS] Generated new screenshot for "${this.id}"`);
+    this.png = mapnik.Image.fromBytes(new Buffer(png, 'base64'));
+    if (SHA) {
+      this.downloadFromGithub();
+    } else {
+      this.compareScreenshots();
+    }
+  }
+
+  /**
+   * Since we are using `git-lfs`, screenshots are not necessarily available within our local
+   * directory.  To get around this, we download the latest screenshot from Github.
+   */
+  downloadFromGithub() {
+    console.info(`[STATUS] Downloading screenshot from Github: ${this.url} => ${this.path}`);
+    child_process.execSync(`curl ${this.url} > "${this.path}"`);
+    this.compareScreenshots();
+  }
+
+  /**
+   * Compares the generated screenshot to the existing screenshot.  If it does not match, an error
+   * will be thrown.
+   */
+  compareScreenshots() {
+    console.info(`[STATUS] Comparing screenshots`);
+    try {
+      let referenceScreenshot = mapnik.Image.open(this.path);
+      this.overwriteExistingScreenshot();
+      if (referenceScreenshot.compare(this.png)) {
+        throw new Error(`screenshot "${this.id}" has changed.`);
+      } else {
+        console.info('[STATUS] Screenshot has not changed');
+      }
+    } catch (e) {
+      console.info(`[STATUS] No reference screenshot found`);
+      this.overwriteExistingScreenshot();
+      throw new Error(`screenshot "${this.id}" was not found.`);
+    }
+  }
+
+  /**
+   * Replaces the existing screenshot with the newly generated one.
+   */
+  overwriteExistingScreenshot() {
+    console.info(`[STATUS] Saving new screenshot`);
+    this.png.save(this.path);
+  }
+}
+
+module.exports = (id) => new Screenshot(id);

e2e/index.e2e.js

@@ -0,0 +1,12 @@
+const screenshot = require('./_screenshot.js');
+
+describe('hello, protractor', function () {
+  describe('index', function () {
+    browser.get('/');
+    it('should have a title', function () {
+      expect(browser.getTitle()).toBe('Material2');
+      screenshot('initial state');
+    });
+  });
+});
+

package.json

@@ -10,10 +10,11 @@
   "scripts": {
     "build": "ng build",
     "dartanalyzer": "cd dist/dart && pub install && cd ../.. && ts-node scripts/ci/dart_analyzer",
-    "demo-app": "cd src && ng serve",
+    "demo-app": "ng serve",
     "test": "karma start test/karma.conf.js",
     "tslint" : "tslint -c tslint.json 'src/**/*.ts'",
-    "typings": "typings install --ambient"
+    "typings": "typings install --ambient",
+    "e2e": "protractor ./protractor.conf.js"
   },
   "version": "2.0.0-alpha.0",
   "license": "MIT",
@@ -24,6 +25,8 @@
     "angular2": "2.0.0-beta.8",
     "es6-promise": "^3.0.2",
     "es6-shim": "^0.33.3",
+    "mapnik": "^3.5.2",
+    "protractor": "^3.1.1",
     "reflect-metadata": "0.1.2",
     "rxjs": "5.0.0-beta.2",
     "systemjs": "0.19.4",

protractor.conf.js

@@ -0,0 +1,15 @@
+var key = require('./scripts/sauce/sauce_config.js');
+
+exports.config = {
+  useAllAngular2AppRoots: true,
+  specs: [ './e2e/**/*.e2e.js' ],
+  baseUrl: 'http://localhost:4200',
+  sauceUser: process.env.SAUCE_USERNAME,
+  sauceKey: key,
+  capabilities: {
+    'browserName': 'chrome',
+    'tunnel-identifier': process.env.TRAVIS_JOB_NUMBER,
+    'build': process.env.TRAVIS_JOB_NUMBER,
+    'name': 'Material 2 E2E Tests'
+  }
+};

screenshots/initial state.screenshot.png

@@ -15,13 +15,17 @@ is_dart && source scripts/ci/sources/env_dart.sh
 
 start_tunnel
 npm run build
+echo
 is_dart && pub install
 
 wait_for_tunnel
 if is_dart; then
   npm run dartanalyzer
+elif [[ "$MODE" = e2e* ]]; then
+  ng serve &
+  sleep 20
+  npm run e2e
 else
   karma start test/karma.conf.js --single-run --no-auto-watch --reporters='dots'
 fi
 teardown_tunnel
-

scripts/ci/build-and-test.sh

@@ -3,7 +3,7 @@
 
 start_tunnel() {
   case "$MODE" in
-    saucelabs*)
+    e2e*|saucelabs*)
       ./scripts/sauce/sauce_connect_setup.sh
       ;;
     browserstack*)
@@ -16,7 +16,7 @@ start_tunnel() {
 
 wait_for_tunnel() {
   case "$MODE" in
-    saucelabs*)
+    e2e*|saucelabs*)
       ./scripts/sauce/sauce_connect_block.sh
       ;;
     browserstack*)
@@ -30,7 +30,7 @@ wait_for_tunnel() {
 
 teardown_tunnel() {
   case "$MODE" in
-    saucelabs*)
+    e2e*|saucelabs*)
       ./scripts/sauce/sauce_connect_teardown.sh
       ;;
     browserstack*)

scripts/ci/sources/tunnel.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+export MODE=e2e
+export LOGS_DIR=/tmp/angular-material2-build/logs
+export BROWSER_PROVIDER_READY_FILE=$LOGS_DIR/sauce-connect-readyfile
+export SAUCE_USERNAME=angular-ci
+export SAUCE_ACCESS_KEY=9b988f434ff8-fbca-8aa4-4ae3-35442987
+export TRAVIS_JOB_NUMBER=12345
+
+mkdir -p $LOGS_DIR
+rm -f $BROWSER_PROVIDER_READY_FILE
+
+# Force cleanup (shouldn't be necessary)
+killall angular-cli
+./scripts/sauce/sauce_connect_teardown.sh
+# Run the script
+./scripts/ci/build-and-test.sh
+# Actual cleanup
+./scripts/sauce/sauce_connect_teardown.sh
+killall angular-cli

scripts/e2e.sh

@@ -0,0 +1 @@
+module.exports = process.env.SAUCE_ACCESS_KEY.split('').reverse().join('');

scripts/sauce/sauce_config.js

@@ -1,10 +1,11 @@
 #!/bin/bash
 
 # Wait for Connect to be ready before exiting
-printf "Connecting to Sauce."
+echo "Connecting to Sauce Labs"
 while [ ! -f $BROWSER_PROVIDER_READY_FILE ]; do
   printf "."
   #dart2js takes longer than the travis 10 min timeout to complete
   sleep .5
 done
+echo
 echo "Connected"

scripts/sauce/sauce_connect_block.sh

@@ -12,22 +12,30 @@ set -e -o pipefail
 # before_script:
 #   - curl https://gist.github.com/santiycr/5139565/raw/sauce_connect_setup.sh | bash
 
-CONNECT_URL="https://saucelabs.com/downloads/sc-4.3.11-linux.tar.gz"
 CONNECT_DIR="/tmp/sauce-connect-$RANDOM"
 CONNECT_DOWNLOAD="sc-latest-linux.tar.gz"
 
 CONNECT_LOG="$LOGS_DIR/sauce-connect"
 CONNECT_STDOUT="$LOGS_DIR/sauce-connect.stdout"
 CONNECT_STDERR="$LOGS_DIR/sauce-connect.stderr"
 
-# Get Connect and start it
+# Get the appropriate URL for downloading Sauce Connect
+if [ `uname -s` = "Darwin" ]; then
+  # If the user is running Mac, download the OSX version
+  # https://en.wikipedia.org/wiki/Darwin_(operating_system)
+  CONNECT_URL="https://saucelabs.com/downloads/sc-4.3.11-osx.zip"
+else
+  # Otherwise, default to Linux for Travis-CI
+  CONNECT_URL="https://saucelabs.com/downloads/sc-4.3.11-linux.tar.gz"
+fi
 mkdir -p $CONNECT_DIR
 cd $CONNECT_DIR
 curl $CONNECT_URL -o $CONNECT_DOWNLOAD 2> /dev/null 1> /dev/null
 mkdir sauce-connect
 tar --extract --file=$CONNECT_DOWNLOAD --strip-components=1 --directory=sauce-connect > /dev/null
 rm $CONNECT_DOWNLOAD
 
+
 SAUCE_ACCESS_KEY=`echo $SAUCE_ACCESS_KEY | rev`
 
 ARGS=""
@@ -45,5 +53,7 @@ echo "Starting Sauce Connect in the background, logging into:"
 echo "  $CONNECT_LOG"
 echo "  $CONNECT_STDOUT"
 echo "  $CONNECT_STDERR"
+echo "  ---"
+echo "  $ARGS"
 sauce-connect/bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY $ARGS \
   --logfile $CONNECT_LOG 2> $CONNECT_STDERR 1> $CONNECT_STDOUT &