feat(smoke-tests): test auto-updating from the version of Compass that was just packaged COMPASS-8536 (#6629)

* Test auto-updating from the version of Compass that was just packaged

Author: lerouxb

.evergreen/functions.yml

@@ -649,6 +649,13 @@ functions:
           npm run --unsafe-perm --workspace compass-e2e-tests test-packaged-ci
 
   smoketest-packaged-app:
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: compass-mongodb-com
+        expansion_name: generated_token
+        permissions:  # optional
+          contents: read
     - command: shell.exec
       # Fail the task if it's idle for 10 mins
       timeout_secs: 600
@@ -667,16 +674,22 @@ functions:
           # Load environment variables
           eval $(.evergreen/print-compass-env.sh)
 
+          npm i -w packages/compass-smoke-tests https://x-access-token:${generated_token}@github.com/10gen/compass-mongodb-com --engine-strict=false
+
           if [[ "$IS_WINDOWS" == "true" ]]; then
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi
+            # TODO: windows_setup
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests=auto-update-from
           fi
 
           if [[ "$IS_OSX" == "true" ]]; then
             echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)"
             export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip
+            # NOTE: We're also skipping auto-update of the macOS app in CI
+            # because it doesn't work. Running a different test to make sure it
+            # can install and run successfully at least.
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query
           fi
 
           #if [[ "$IS_UBUNTU" == "true" ]]; then

.evergreen/preinstall.sh

@@ -13,9 +13,9 @@ echo "APPDATA: $APPDATA"
 echo "PATH: $PATH"
 
 # these are super useful if you want to run the smoke tests locally
-echo "DEV_VERSION_IDENTIFIER: $DEV_VERSION_IDENTIFIER"
-echo "EVERGREEN_BUCKET_NAME: $EVERGREEN_BUCKET_NAME"
-echo "EVERGREEN_BUCKET_KEY_PREFIX: $EVERGREEN_BUCKET_KEY_PREFIX"
+echo "export DEV_VERSION_IDENTIFIER=$DEV_VERSION_IDENTIFIER"
+echo "export EVERGREEN_BUCKET_KEY_PREFIX=$EVERGREEN_BUCKET_KEY_PREFIX"
+echo "export EVERGREEN_BUCKET_NAME=$EVERGREEN_BUCKET_NAME"
 
 echo "IS_OSX: $IS_OSX"
 echo "IS_LINUX: $IS_LINUX"

packages/compass-e2e-tests/helpers/selectors.ts

@@ -1380,3 +1380,12 @@ export const GlobalWrites = {
   SampleFindingDocuments: '[data-testid="sample-finding-documents"]',
   SampleInsertingDocuments: '[data-testid="sample-inserting-documents"]',
 };
+
+// Auto-update toasts
+export const AutoUpdateToast = '[data-testid="toast-compass-update"]';
+export const AutoUpdateRestartButton =
+  '[data-testid="auto-update-restart-button"]';
+export const AutoUpdateDownloadLink =
+  '[data-testid="auto-update-download-link"]';
+export const AutoUpdateReleaseNotesLink =
+  '[data-testid="auto-update-release-notes-link"]';

packages/compass-e2e-tests/tests/auto-update.test.ts

@@ -0,0 +1,78 @@
+import { expect } from 'chai';
+import {
+  init,
+  cleanup,
+  Selectors,
+  screenshotPathName,
+} from '../helpers/compass';
+
+function wait(ms: number) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+describe('Auto-update', function () {
+  it('auto-update from', async function () {
+    if (process.env.TEST_NAME !== 'auto-update-from') {
+      // we don't want this test to execute along with all the others under
+      // normal circumstances because it is destructive - it overwrites Compass
+      // itself
+      this.skip();
+    }
+
+    // run the app and wait for it to auto-update
+    console.log('starting compass the first time');
+    const compass = await init('auto-update from', { firstRun: true });
+    const { browser } = compass;
+    try {
+      await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
+
+      if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+        const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
+        await restartButton.waitForDisplayed();
+
+        // We could click the restart button to apply the update and restart the
+        // app, but restarting the app confuses webdriverio or at least our test
+        // helpers. So we're going to just restart the app manually.
+        await browser.pause(1000);
+      } else {
+        // When auto-update is not supported the toast contains a link to
+        // download
+        const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
+        await linkElement.waitForDisplayed();
+        expect(await linkElement.getAttribute('href')).to.equal(
+          'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
+        );
+      }
+    } finally {
+      await browser.screenshot(screenshotPathName('auto-update-from'));
+      await cleanup(compass);
+    }
+
+    if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+      console.log(
+        'pause to make sure the app properly exited before starting again'
+      );
+      await wait(10_000);
+
+      console.log('starting compass a second time');
+      // run the app again and check that the version changed
+      const compass = await init('auto-update from restart', {
+        firstRun: false,
+      });
+      const { browser } = compass;
+      try {
+        await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
+        await browser
+          .$(Selectors.AutoUpdateReleaseNotesLink)
+          .waitForDisplayed();
+      } finally {
+        await browser.screenshot(
+          screenshotPathName('auto-update-from-restart')
+        );
+        await cleanup(compass);
+      }
+    }
+  });
+});

packages/compass-smoke-tests/.depcheckrc

@@ -1,3 +1,4 @@
 ignores:
  - '@mongodb-js/prettier-config-compass'
  - '@mongodb-js/tsconfig-compass'
+ - compass-mongodb-com

packages/compass-smoke-tests/src/cli.ts

@@ -2,11 +2,12 @@
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import path from 'node:path';
+import { once } from 'node:events';
 
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import { pick } from 'lodash';
-import { execute } from './execute';
+import { execute, executeAsync } from './execute';
 import {
   type PackageDetails,
   readPackageDetails,
@@ -25,6 +26,8 @@ import { installWindowsMSI } from './installers/windows-msi';
 const SUPPORTED_PLATFORMS = ['win32', 'darwin', 'linux'] as const;
 const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;
 
+const SUPPORTED_TESTS = ['time-to-first-query', 'auto-update-from'] as const;
+
 function isSupportedPlatform(
   value: unknown
 ): value is typeof SUPPORTED_PLATFORMS[number] {
@@ -104,7 +107,14 @@ const argv = yargs(hideBin(process.argv))
     type: 'boolean',
     description: 'Do not delete the sandbox after a run',
     default: false,
-  });
+  })
+  .option('tests', {
+    type: 'array',
+    string: true,
+    choices: SUPPORTED_TESTS,
+    description: 'Which tests to run',
+  })
+  .default('tests', SUPPORTED_TESTS.slice());
 
 type TestSubject = PackageDetails & {
   filepath: string;
@@ -184,23 +194,47 @@ async function run() {
       'platform',
       'arch',
       'package',
+      'tests',
     ])
   );
 
-  const { kind, filepath, appName } = await getTestSubject(context);
+  const { kind, appName, filepath, autoUpdatable } = await getTestSubject(
+    context
+  );
   const install = getInstaller(kind);
 
   try {
-    const { appPath, uninstall } = install({
-      appName,
-      filepath,
-      destinationPath: context.sandboxPath,
-    });
+    if (context.tests.length === 0) {
+      console.log('Warning: not performing any tests. Pass --tests.');
+    }
 
-    try {
-      runTest({ appName, appPath });
-    } finally {
-      await uninstall();
+    for (const testName of context.tests) {
+      const { appPath, uninstall } = install({
+        appName,
+        filepath,
+        destinationPath: context.sandboxPath,
+      });
+
+      try {
+        if (testName === 'time-to-first-query') {
+          // Auto-update does not work on mac in CI at the moment. So in that case
+          // we just run the E2E tests to make sure the app at least starts up.
+          runTimeToFirstQuery({
+            appName,
+            appPath,
+          });
+        }
+        if (testName === 'auto-update-from') {
+          await runUpdateTest({
+            appName,
+            appPath,
+            autoUpdatable,
+            testName,
+          });
+        }
+      } finally {
+        await uninstall();
+      }
     }
   } finally {
     if (context.skipCleanup) {
@@ -212,12 +246,30 @@ async function run() {
   }
 }
 
-type RunTestOptions = {
+async function importUpdateServer() {
+  try {
+    return (await import('compass-mongodb-com')).default;
+  } catch (err: unknown) {
+    console.log('Remember to npm link compass-mongodb-com');
+    throw err;
+  }
+}
+
+async function startAutoUpdateServer() {
+  console.log('Starting auto-update server');
+  const { httpServer, updateChecker, start } = (await importUpdateServer())();
+  start();
+  await once(updateChecker, 'refreshed');
+
+  return httpServer;
+}
+
+type RunE2ETestOptions = {
   appName: string;
   appPath: string;
 };
 
-function runTest({ appName, appPath }: RunTestOptions) {
+function runTimeToFirstQuery({ appName, appPath }: RunE2ETestOptions) {
   execute(
     'npm',
     [
@@ -241,6 +293,63 @@ function runTest({ appName, appPath }: RunTestOptions) {
   );
 }
 
+type RunUpdateTestOptions = {
+  appName: string;
+  appPath: string;
+  autoUpdatable?: boolean;
+  testName: string;
+};
+
+async function runUpdateTest({
+  appName,
+  appPath,
+  autoUpdatable,
+  testName,
+}: RunUpdateTestOptions) {
+  process.env.PORT = '0'; // dynamic port
+  process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';
+
+  const server = await startAutoUpdateServer();
+
+  const address = server.address();
+  assert(typeof address === 'object' && address !== null);
+  const port = address.port;
+  const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
+  console.log({ HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE });
+
+  try {
+    // must be async because the update server is running in the same process
+    await executeAsync(
+      'npm',
+      [
+        'run',
+        '--unsafe-perm',
+        'test-packaged',
+        '--workspace',
+        'compass-e2e-tests',
+        '--',
+        '--test-filter=auto-update',
+      ],
+      {
+        // We need to use a shell to get environment variables setup correctly
+        shell: true,
+        env: {
+          ...process.env,
+          HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
+          AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
+          TEST_NAME: testName,
+          COMPASS_APP_NAME: appName,
+          COMPASS_APP_PATH: appPath,
+        },
+      }
+    );
+  } finally {
+    console.log('Stopping auto-update server');
+    server.close();
+    delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
+  }
+}
+
 run()
   .then(function () {
     console.log('done');

packages/compass-smoke-tests/src/compass-mongodb-com.d.ts

@@ -0,0 +1,9 @@
+declare module 'compass-mongodb-com' {
+  import type http from 'http';
+  function updateServer(): {
+    start: () => void;
+    httpServer: http.Server;
+    updateChecker: NodeJS.EventEmitter;
+  };
+  export = updateServer;
+}

packages/compass-smoke-tests/src/context.ts

@@ -9,5 +9,6 @@ export type SmokeTestsContext = {
   forceDownload?: boolean;
   localPackage?: boolean;
   sandboxPath: string;
+  tests: string[];
   skipCleanup: boolean;
 };