ci: set up lighthouse

- define CircleCI version
- update NodeJS docker image
- use the branch as part of the cache_key
- add the circleci/build-tools orb
- store the build output for reuse in other jobs
- add new job to run the lighthouse a11y audits on various pages
- add new job to run all of the lighthouse audits on the home page
- add typing and comments to the original scripts
- update to the latest firebase-tools

Author: Splaktar

.circleci/config.yml

@@ -1,6 +1,18 @@
+# Note: YAML anchors allow an object to be re-used, reducing duplication.
+# The ampersand declares an alias for an object, then later the `<<: *name`
+# syntax dereferences it.
+# See https://blog.daemonl.com/2016/02/yaml.html
+# To validate changes, use an online parser, eg.
+# https://yaml-online-parser.appspot.com/
+
+# CircleCI configuration version
+# Version 2.1 allows for extra config reuse features
+# https://circleci.com/docs/2.0/reusing-config/#getting-started-with-config-reuse
+version: 2.1
+
 # Cache key for CircleCI. We want to invalidate the cache whenever the Yarn lock file changes.
-var_1: &cache_key material-angular-io-{{ checksum "yarn.lock" }}
-var_2: &default_docker_image circleci/node:12.9.1-browsers
+var_1: &cache_key material-angular-io-{{ .Branch }}-{{ checksum "yarn.lock" }}
+var_2: &default_docker_image circleci/node:12.14.1-browsers
 
 # Settings common to each job
 var_3: &job_defaults
@@ -19,19 +31,41 @@ var_5: &yarn_install
     name: "Installing project dependencies"
     command: yarn install --frozen-lockfile --non-interactive
 
-# Job step for checking out the source code from GitHub. This also ensures that the source code
-# is rebased on top of master.
-var_6: &checkout_code
-  checkout:
-    # After checkout, rebase on top of master. By default, PRs are not rebased on top of master,
-    # which we want. See https://discuss.circleci.com/t/1662
-    post: git pull --ff-only origin "refs/pull/${CI_PULL_REQUEST//*pull\//}/merge"
+# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
+# https://circleci.com/blog/deep-diving-into-circleci-workspaces/
+var_6: &workspace_location ~/
+
+orbs:
+  build-tools: circleci/[email protected]
+
+commands:
+  store_build_output:
+    description: 'Stores build artifacts'
+    steps:
+      - run:
+          name: Move compiled apps to workspace
+          command: |
+            set -exu
+            mkdir -p ~/dist
+            mv dist/* ~/dist/
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist
+  # Command for checking out the source code from GitHub. This also ensures that the source code
+  # can be merged to the master branch without conflicts.
+  checkout_and_rebase:
+    description: Checkout and verify clean merge with master
+    steps:
+      - checkout
+      - build-tools/merge-with-parent:
+          parent: master
 
 jobs:
   lint:
     <<: *job_defaults
     steps:
-    - *checkout_code
+    - checkout_and_rebase
     - restore_cache:
         key: *cache_key
     - *yarn_install
@@ -45,26 +79,55 @@ jobs:
     # https://circleci.com/docs/2.0/configuration-reference/#resource_class
     resource_class: large
     steps:
-    - *checkout_code
+    - checkout_and_rebase
     - restore_cache:
         key: *cache_key
     - *yarn_install
     - run: yarn prod-build
     - *save_cache
+    - store_build_output
 
   test:
     <<: *job_defaults
     steps:
-    - *checkout_code
+    - checkout_and_rebase
     - restore_cache:
         key: *cache_key
     - *yarn_install
     - run: yarn test --watch false --progress=false
 
+  audit_pages_for_a11y:
+    <<: *job_defaults
+    steps:
+      - attach_workspace:
+          at: *workspace_location
+      - checkout_and_rebase
+      - restore_cache:
+          key: *cache_key
+      - *yarn_install
+      - run: yarn test:audit:a11y:ci
+
+  lighthouse_audits:
+    <<: *job_defaults
+    steps:
+      - attach_workspace:
+          at: *workspace_location
+      - checkout_and_rebase
+      - restore_cache:
+          key: *cache_key
+      - *yarn_install
+      - run: yarn test:audit:ci
+
 workflows:
   version: 2
   default_workflow:
     jobs:
     - lint
     - build
     - test
+    - audit_pages_for_a11y:
+        requires:
+          - build
+    - lighthouse_audits:
+        requires:
+          - build

package.json

@@ -9,6 +9,7 @@
     "start:jit": "ng serve --aot=false",
     "start:prod": "ng serve --prod",
     "start:scenes": "ng serve scenes",
+    "start:static": "firebase serve --only hosting --port 4200",
     "lint": "ng lint",
     "test": "ng test",
     "pree2e": "webdriver-manager update",
@@ -22,7 +23,14 @@
     "postinstall": "webdriver-manager update --gecko false && ngcc --properties es2015 browser module main --first-only --create-ivy-entry-points",
     "publish-prod": "bash ./tools/deploy.sh stable prod",
     "publish-dev": "bash ./tools/deploy.sh",
-    "publish-beta": "bash ./tools/deploy.sh stable beta"
+    "publish-beta": "bash ./tools/deploy.sh stable beta",
+    "test:audit:a11y": "node tools/audit-docs-a11y",
+    "test:audit:a11y:localhost": "run-p --race \"~~light-server -s dist/material-angular-io -p 4200 --quiet\" \"test:audit:a11y http://localhost:4200\" --",
+    "test:audit:a11y:ci": "run-p --race \"~~light-server -s ../dist/material-angular-io -p 4200 --quiet\" \"test:audit:a11y http://localhost:4200\" --",
+    "test:audit": "node tools/audit-docs",
+    "test:audit:localhost": "run-p --race \"start:static\" \"test:audit http://localhost:4200 2000\" --",
+    "test:audit:ci": "run-p --race \"~~light-server -s ../dist/material-angular-io -p 4200 --quiet\" \"test:audit http://localhost:4200\" --",
+    "~~light-server": "light-server --bind=localhost --historyindex=/index.html --no-reload"
   },
   "private": true,
   "dependencies": {
@@ -55,8 +63,9 @@
     "@angular/localize": "^11.0.0",
     "@types/jasmine": "^3.6.0",
     "@types/node": "^12.12.70",
+    "@types/shelljs": "~0.8.8",
     "codelyzer": "^6.0.1",
-    "firebase-tools": "^8.16.0",
+    "firebase-tools": "^9.2.1",
     "jasmine-core": "^3.6.0",
     "jasmine-spec-reporter": "^5.0.2",
     "karma": "~5.1.1",
@@ -65,7 +74,13 @@
     "karma-firefox-launcher": "^2.1.0",
     "karma-jasmine": "^4.0.1",
     "karma-jasmine-html-reporter": "^1.5.4",
+    "light-server": "^2.6.2",
+    "lighthouse": "~7.0.0",
+    "lighthouse-logger": "~1.2.0",
+    "npm-run-all": "^4.1.5",
     "protractor": "^7.0.0",
+    "puppeteer": "5.4.1",
+    "shelljs": "^0.8.4",
     "ts-node": "^8.10.1",
     "tslint": "^6.1.2",
     "typescript": "4.0.2"

tools/audit-docs-a11y.js

@@ -0,0 +1,46 @@
+#!/bin/env node
+'use strict';
+
+/**
+ * Originally from https://github.com/angular/angular/blob/b48eabddb83900496e5e00fb98e192c6c49dff26/aio/scripts/test-aio-a11y.js
+ *
+ * Usage:
+ * ```sh
+ * node tools/audit-docs-a11y <origin>
+ * ```
+ *
+ * Runs accessibility audits on several (pre-defined) pages on the specified
+ * origin. It fails, if the score for any page is below the minimum (see
+ * `MIN_SCORES_PER_PAGE` below).
+ *
+ * `<origin>` is the origin (scheme + hostname + port) of an material.angular.io
+ * deployment. It can be remote (e.g. `https://next.material.angular.io`) or local (e.g.
+ * `http://localhost:4200`).
+ */
+
+// Imports
+const sh = require('shelljs');
+sh.set('-e');
+
+// Constants
+const MIN_SCORES_PER_PAGE = {
+  '' : 100,
+  'components/categories' : 91,
+  'cdk/categories' : 91,
+  'guides' : 100,
+  'guide/creating-a-custom-form-field-control' : 99,
+  'guide/getting-started' : 98,
+  'cdk/a11y/overview' : 85,
+  'cdk/a11y/api' : 89,
+  'cdk/a11y/examples' : 85,
+  'components/button/overview' : 92,
+  'components/button/api' : 89,
+  'components/button/examples' : 90,
+};
+
+// Run the a11y audit against the above pages
+const lighthouseAuditCmd = `"${process.execPath}" "${__dirname}/lighthouse-audit"`;
+const origin = process.argv[2];
+for (const [page, minScore] of Object.entries(MIN_SCORES_PER_PAGE)) {
+  sh.exec(`${lighthouseAuditCmd} ${origin}/${page} accessibility:${minScore}`);
+}

tools/audit-docs.js

@@ -0,0 +1,72 @@
+#!/bin/env node
+'use strict';
+
+/**
+ * Usage:
+ * ```sh
+ * node tools/audit-docs <origin> <delay>
+ * ```
+ *
+ * Runs the configured audits on several (pre-defined) pages on the specified
+ * origin. It fails if any score for any page is below the minimum (see
+ * `MIN_SCORES_PER_PAGE` below).
+ *
+ * `<origin>` is the origin (scheme + hostname + port) of an material.angular.io
+ *  deployment. It can be remote (e.g. `https://next.material.angular.io`) or local (e.g.
+ *  `http://localhost:4200`).
+ *
+ * `<delay>` is a millisecond value used with `setTimeout()` to allow a configurable delay
+ *  for the HTTP server to start up. This is needed when used with `firebase serve`.
+ */
+
+// Imports
+const sh = require('shelljs');
+sh.set('-e');
+
+// Constants
+/**
+ * @type {{minScores: {performance: number, accessibility: number, 'best-practices': number, pwa: number, seo: number}, url: string}[]}
+ */
+const MIN_SCORES_PER_PAGE = [
+  {
+    url: '',
+    minScores: {
+      'pwa': 70,
+      'performance': 21,
+      'seo': 98,
+      'best-practices': 100,
+      'accessibility': 100
+    }
+  }
+];
+
+/**
+ * @param {{performance?: number, accessibility?: number, 'best-practices'?: number, pwa?: number, seo?: number}} scores
+ * @returns string scores formatted as described in the docs of lighthouse-audit.js's _main()
+ */
+function formatScores(scores) {
+  let formattedScores = '';
+  Object.keys(scores).map((key, index) => {
+    if (index > 0) {
+      formattedScores += ',';
+    }
+    formattedScores += `${key}:${scores[key]}`;
+  });
+  return formattedScores;
+}
+
+// Run the a11y audit against the above pages
+const lighthouseAuditCmd = `"${process.execPath}" "${__dirname}/lighthouse-audit"`;
+const origin = process.argv[2];
+const delay = process.argv[3];
+if (delay) {
+  setTimeout(_main, delay);
+} else {
+  _main();
+}
+
+function _main() {
+  MIN_SCORES_PER_PAGE.map((urlsAndScores) => {
+    sh.exec(`${lighthouseAuditCmd} ${origin}/${urlsAndScores.url} ${formatScores(urlsAndScores.minScores)}`);
+  });
+}