[Auth] Add some initial WebDriver tests against the emulator (#4580)

* Add webdriver tests as well as updated readme

* Formatting

* Fix markup

* Move location of integration command in package.json

* PR feedback, use emulators:exec instead

* Formatting

* Clean up the test infra a bit

* Formatting

* Update packages-exp/auth-exp/README.md

Co-authored-by: Yuchen Shi <[email protected]>

* PR feedback

Co-authored-by: Yuchen Shi <[email protected]>

Author: sam-gc

packages-exp/auth-exp/README.md

@@ -2,4 +2,55 @@
 
 This is the Firebase Authentication component of the Firebase JS SDK.
 
-**This package is not intended for direct usage, and should only be used via the officially supported [firebase](https://www.npmjs.com/package/firebase) package.**
+**This package is not intended for direct usage, and should only be used via the officially supported [firebase](https://www.npmjs.com/package/firebase) package.**
+
+## Testing
+
+The modular Auth SDK has both unit tests and integration tests, along with a
+host of npm scripts to run these tests. The most important commands are:
+
+| Command | Description |
+| ------- | ----------- |
+| `yarn test` | This will run lint, unit tests, and integration tests against the live environment|
+| `yarn test:<platform>` | Runs all browser tests, unit and integration |
+| `yarn test:<platform>:unit` | Runs only \<platform> unit tests |
+| `yarn test:<platform>:unit:debug` | Runs \<platform> unit tests, auto-watching for file system changes |
+| `yarn test:<platform>:integration` | Runs only integration tests against the live environment |
+| `yarn test:<platform>:integration:local` | Runs all headless \<platform> integration tests against the emulator (more below) |
+
+Where \<platform> is "browser" or "node". There are also cordova tests, but they
+are not broken into such granular details. Check out `package.json` for more.
+
+### Integration testing with the emulator
+
+To test against the emulator, set up the Auth emulator
+([instructions](https://firebase.google.com/docs/emulator-suite/connect_and_prototype)).
+The easiest way to run these tests is to use the `firebase emulators:exec`
+command
+([documentation](https://firebase.google.com/docs/emulator-suite/install_and_configure#startup)).
+You can also manually start the emulator separately, and then point the tests
+to it by setting the `GCLOUD_PROJECT` and `FIREBASE_AUTH_EMULATOR_HOST`
+environmental variables. In addition to the commands listed above, the below
+commands also run various tests:
+
+  * `yarn test:integration:local` — Executes Node and browser emulator
+    integration tests, as well as the Selenium WebDriver tests
+  
+  * `yarn test:webdriver` — Executes only the Selenium WebDriver
+    integration tests
+
+For example, to run all integration and WebDriver tests against the emulator,
+you would simply execute the following command:
+
+```sh
+firebase emulators:exec --project foo-bar --only auth "yarn test:integration:local"
+```
+
+### Selenium Webdriver tests
+
+These tests assume that you have both Firefox and Chrome installed on your
+computer and in your `$PATH`. The tests will error out if this is not the case.
+The WebDriver tests talk to the emulator, but unlike the headless integration
+tests, these run in a browser robot environment; the assertions themselves run
+in Node. When you run these tests a small Express server will be started to
+serve the static files the browser robot uses.

packages-exp/auth-exp/karma.conf.js

@@ -37,8 +37,8 @@ function getTestFiles(argv) {
     return ['src/**/*.test.ts', 'test/helpers/**/*.test.ts'];
   } else if (argv.integration) {
     return argv.local
-      ? ['test/integration/**/*.test.ts']
-      : ['test/integration/**/*!(local).test.ts'];
+      ? ['test/integration/flows/*.test.ts']
+      : ['test/integration/flows/*!(local).test.ts'];
   } else if (argv.cordova) {
     return ['src/platform_cordova/**/*.test.ts'];
   } else {
@@ -57,13 +57,22 @@ function getClientConfig(argv) {
     return {};
   }
 
+  if (!process.env.GCLOUD_PROJECT || !process.env.FIREBASE_AUTH_EMULATOR_HOST) {
+    console.error(
+      'Local testing against emulator requested, but ' +
+        'GCLOUD_PROJECT and FIREBASE_AUTH_EMULATOR_HOST env variables ' +
+        'are missing'
+    );
+    process.exit(1);
+  }
+
   return {
     authAppConfig: {
       apiKey: 'local-api-key',
-      projectId: 'test-emulator',
+      projectId: process.env.GCLOUD_PROJECT,
       authDomain: 'local-auth-domain'
     },
-    authEmulatorPort: '9099'
+    authEmulatorHost: process.env.FIREBASE_AUTH_EMULATOR_HOST
   };
 }
 

packages-exp/auth-exp/package.json

@@ -25,6 +25,7 @@
     "test": "run-p lint test:all",
     "test:all": "run-p test:browser test:node",
     "test:ci": "node ../../scripts/run_tests_in_ci.js -s test:all",
+    "test:integration:local": "run-s test:node:integration:local test:browser:integration:local test:webdriver",
     "test:browser": "karma start --single-run",
     "test:browser:unit": "karma start --single-run --unit",
     "test:browser:integration": "karma start --single-run --integration",
@@ -37,6 +38,7 @@
     "test:node:unit": "node ./scripts/run-node-tests.js",
     "test:node:integration": "node ./scripts/run-node-tests.js --integration",
     "test:node:integration:local": "node ./scripts/run-node-tests.js --integration --local",
+    "test:webdriver": "rollup -c test/integration/webdriver/static/rollup.config.js && node ./scripts/run-node-tests.js --webdriver",
     "api-report": "api-extractor run --local --verbose",
     "predoc": "node ../../scripts/exp/remove-exp.js temp",
     "doc": "api-documenter markdown --input temp --output docs",
@@ -47,17 +49,18 @@
     "@firebase/app-exp": "0.x"
   },
   "dependencies": {
+    "@firebase/component": "0.2.0",
     "@firebase/logger": "0.2.6",
     "@firebase/util": "0.3.4",
-    "@firebase/component": "0.2.0",
     "node-fetch": "2.6.1",
+    "selenium-webdriver": "4.0.0-beta.1",
     "tslib": "^2.0.0"
   },
   "license": "Apache-2.0",
   "devDependencies": {
     "@firebase/app-exp": "0.0.900",
-    "rollup": "2.35.1",
     "@rollup/plugin-json": "4.1.0",
+    "rollup": "2.35.1",
     "rollup-plugin-sourcemaps": "0.6.3",
     "rollup-plugin-typescript2": "0.29.0",
     "@rollup/plugin-strip": "2.0.0",

packages-exp/auth-exp/scripts/run-node-tests.js

@@ -30,7 +30,8 @@ var child_process_promise_1 = require('child-process-promise');
 var yargs = require('yargs');
 var argv = yargs.options({
   local: { type: 'boolean' },
-  integration: { type: 'boolean' }
+  integration: { type: 'boolean' },
+  webdriver: { type: 'boolean' }
 }).argv;
 var nyc = path_1.resolve(__dirname, '../../../node_modules/.bin/nyc');
 var mocha = path_1.resolve(__dirname, '../../../node_modules/.bin/mocha');
@@ -45,20 +46,33 @@ if (argv.integration) {
   if (argv.local) {
     testConfig.push('test/integration/flows/*.local.test.ts');
   }
+} else if (argv.webdriver) {
+  testConfig = ['test/integration/webdriver/**.test.ts', '--delay'];
 }
 var args = __spreadArrays(['--reporter', 'lcovonly', mocha], testConfig, [
   '--config',
   '../../config/mocharc.node.js'
 ]);
 if (argv.local) {
-  process.env.AUTH_EMULATOR_PORT = '9099';
-  process.env.AUTH_EMULATOR_PROJECT_ID = 'test-emulator';
+  if (!process.env.GCLOUD_PROJECT || !process.env.FIREBASE_AUTH_EMULATOR_HOST) {
+    console.error(
+      'Local testing against emulator requested, but ' +
+        'GCLOUD_PROJECT and FIREBASE_AUTH_EMULATOR_HOST env variables ' +
+        'are missing'
+    );
+    process.exit(1);
+  }
 }
 args = args.concat(argv._);
-var childProcess = child_process_promise_1.spawn(nyc, args, {
+var spawned = child_process_promise_1.spawn(nyc, args, {
   stdio: 'inherit',
   cwd: process.cwd()
-}).childProcess;
+});
+var childProcess = spawned.childProcess;
+spawned['catch'](function () {
+  childProcess.kill();
+  process.exit(1);
+});
 process.once('exit', function () {
   return childProcess.kill();
 });

packages-exp/auth-exp/scripts/run-node-tests.ts

@@ -26,6 +26,9 @@ const argv = yargs.options({
   },
   integration: {
     type: 'boolean'
+  },
+  webdriver: {
+    type: 'boolean'
   }
 }).argv;
 
@@ -45,6 +48,8 @@ if (argv.integration) {
   if (argv.local) {
     testConfig.push('test/integration/flows/*.local.test.ts');
   }
+} else if (argv.webdriver) {
+  testConfig = ['test/integration/webdriver/**.test.ts', '--delay'];
 }
 
 let args = [
@@ -56,17 +61,30 @@ let args = [
   '../../config/mocharc.node.js'
 ];
 
+// Make sure that the environment variables are present for local test
 if (argv.local) {
-  process.env.AUTH_EMULATOR_PORT = '9099';
-  process.env.AUTH_EMULATOR_PROJECT_ID = 'test-emulator';
+  if (!process.env.GCLOUD_PROJECT || !process.env.FIREBASE_AUTH_EMULATOR_HOST) {
+    console.error(
+      'Local testing against emulator requested, but ' +
+        'GCLOUD_PROJECT and FIREBASE_AUTH_EMULATOR_HOST env variables ' +
+        'are missing'
+    );
+    process.exit(1);
+  }
 }
 
 args = args.concat(argv._ as string[]);
 
-const childProcess = spawn(nyc, args, {
+const spawned = spawn(nyc, args, {
   stdio: 'inherit',
   cwd: process.cwd()
-}).childProcess;
+});
+
+const childProcess = spawned.childProcess;
+spawned.catch(() => {
+  childProcess.kill();
+  process.exit(1);
+});
 
 process.once('exit', () => childProcess.kill());
 process.once('SIGINT', () => childProcess.kill('SIGINT'));

packages-exp/auth-exp/test/helpers/integration/emulator_rest_helpers.ts

@@ -15,11 +15,9 @@
  * limitations under the License.
  */
 
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { Auth } from '@firebase/auth-exp';
-import { getApps } from '@firebase/app-exp';
 import { FetchProvider } from '../../../src/core/util/fetch_provider';
 import * as fetchImpl from 'node-fetch';
+import { getAppConfig, getEmulatorUrl } from './settings';
 
 if (typeof document !== 'undefined') {
   FetchProvider.initialize(fetch);
@@ -52,11 +50,10 @@ interface OobCodesResponse {
   oobCodes: OobCodeSession[];
 }
 
-export async function getPhoneVerificationCodes(
-  auth: Auth
-): Promise<Record<string, VerificationSession>> {
-  assertEmulator(auth);
-  const url = getEmulatorUrl(auth, 'verificationCodes');
+export async function getPhoneVerificationCodes(): Promise<
+  Record<string, VerificationSession>
+> {
+  const url = buildEmulatorUrlForPath('verificationCodes');
   const response: VerificationCodesResponse = await (
     await FetchProvider.fetch()(url)
   ).json();
@@ -67,27 +64,21 @@ export async function getPhoneVerificationCodes(
   }, {} as Record<string, VerificationSession>);
 }
 
-export async function getOobCodes(auth: Auth): Promise<OobCodeSession[]> {
-  assertEmulator(auth);
-  const url = getEmulatorUrl(auth, 'oobCodes');
+export async function getOobCodes(): Promise<OobCodeSession[]> {
+  const url = buildEmulatorUrlForPath('oobCodes');
   const response: OobCodesResponse = await (
     await FetchProvider.fetch()(url)
   ).json();
   return response.oobCodes;
 }
 
-function getEmulatorUrl(auth: Auth, endpoint: string): string {
-  const { host, port, protocol } = auth.emulatorConfig!;
-  const projectId = getProjectId(auth);
-  return `${protocol}://${host}:${port}/emulator/v1/projects/${projectId}/${endpoint}`;
+export async function resetEmulator(): Promise<void> {
+  const url = buildEmulatorUrlForPath('accounts');
+  await FetchProvider.fetch()(url, { method: 'DELETE' });
 }
 
-function getProjectId(auth: Auth): string {
-  return getApps().find(app => app.name === auth.name)!.options.projectId!;
-}
-
-function assertEmulator(auth: Auth): void {
-  if (!auth.emulatorConfig) {
-    throw new Error("Can't fetch OOB codes against prod API");
-  }
+function buildEmulatorUrlForPath(endpoint: string): string {
+  const emulatorBaseUrl = getEmulatorUrl();
+  const projectId = getAppConfig().projectId;
+  return `${emulatorBaseUrl}/emulator/v1/projects/${projectId}/${endpoint}`;
 }

packages-exp/auth-exp/test/helpers/integration/helpers.ts

@@ -22,6 +22,7 @@ import { Auth, User } from '../../../src/model/public_types';
 import { getAuth, useAuthEmulator } from '../../../'; // Use browser OR node dist entrypoint depending on test env.
 import { _generateEventId } from '../../../src/core/util/event_id';
 import { getAppConfig, getEmulatorUrl } from './settings';
+import { resetEmulator } from './emulator_rest_helpers';
 
 interface IntegrationTestAuth extends Auth {
   cleanUp(): Promise<void>;
@@ -55,12 +56,17 @@ export function getTestInstance(requireEmulator = false): Auth {
   });
 
   auth.cleanUp = async () => {
-    // Clear out any new users that were created in the course of the test
-    for (const user of createdUsers) {
-      try {
-        await user.delete();
-      } catch {
-        // Best effort. Maybe the test already deleted the user ¯\_(ツ)_/¯
+    // If we're in an emulated environment, the emulator will clean up for us
+    if (emulatorUrl) {
+      await resetEmulator();
+    } else {
+      // Clear out any new users that were created in the course of the test
+      for (const user of createdUsers) {
+        try {
+          await user.delete();
+        } catch {
+          // Best effort. Maybe the test already deleted the user ¯\_(ツ)_/¯
+        }
       }
     }
 

packages-exp/auth-exp/test/helpers/integration/settings.ts

@@ -24,10 +24,10 @@ declare const __karma__: any;
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const PROJECT_CONFIG = require('../../../../../config/project.json');
 
-const EMULATOR_PORT = process.env.AUTH_EMULATOR_PORT;
-const EMULATOR_PROJECT_ID = process.env.AUTH_EMULATOR_PROJECT_ID;
+const EMULATOR_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST;
+const EMULATOR_PROJECT_ID = process.env.GCLOUD_PROJECT;
 
-export const USE_EMULATOR = !!EMULATOR_PORT;
+export const USE_EMULATOR = !!EMULATOR_HOST;
 
 export const PROJECT_ID = USE_EMULATOR
   ? EMULATOR_PROJECT_ID
@@ -52,11 +52,10 @@ export function getAppConfig(): FirebaseOptions {
 
 export function getEmulatorUrl(): string | null {
   // Check karma first, then fallback on node process
-  const emulatorPort: string | null =
-    getKarma()?.config?.authEmulatorPort ||
-    (USE_EMULATOR ? EMULATOR_PORT : null);
-
-  return emulatorPort ? `http://localhost:${emulatorPort}` : null;
+  const host =
+    getKarma()?.config?.authEmulatorHost ||
+    (USE_EMULATOR ? EMULATOR_HOST : null);
+  return host ? `http://${host}` : null;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any

packages-exp/auth-exp/test/integration/flows/oob.local.test.ts

@@ -77,7 +77,7 @@ describe('Integration test: oob codes', () => {
   });
 
   async function code(toEmail: string): Promise<OobCodeSession> {
-    const codes = await getOobCodes(auth);
+    const codes = await getOobCodes();
     return codes.reverse().find(({ email }) => email === toEmail)!;
   }
 

packages-exp/auth-exp/test/integration/flows/phone.test.ts

@@ -88,7 +88,7 @@ describe('Integration test: phone auth', () => {
     fallback: string
   ): Promise<string> {
     if (auth.emulatorConfig) {
-      const codes = await getPhoneVerificationCodes(auth);
+      const codes = await getPhoneVerificationCodes();
       const vid = typeof crOrId === 'string' ? crOrId : crOrId.verificationId;
       return codes[vid].code;
     }