test: adds tests for atlas data lake

NODE-2691

Author: Thomas Reggi

.evergreen/config.yml

@@ -83,6 +83,18 @@ functions:
     - command: expansions.update
       params:
         file: mo-expansion.yml
+  bootstrap mongohoused:
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
+    - command: shell.exec
+      params:
+        background: true
+        script: |
+          ${PREPARE_SHELL}
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
   run tests:
     - command: shell.exec
       type: test
@@ -189,6 +201,15 @@ functions:
         script: |
           MONGODB_URI='${plain_auth_mongodb_uri}' NODE_LTS_NAME='${NODE_LTS_NAME}' \
           bash ${PROJECT_DIRECTORY}/.evergreen/run-ldap-tests.sh
+  run data lake tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        script: |
+          ${PREPARE_SHELL}
+          MONGODB_URI='mongodb://mhuser:pencil@localhost' NODE_LTS_NAME='${NODE_LTS_NAME}' \
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-data-lake-tests.sh
   run tls tests:
     - command: shell.exec
       type: test
@@ -723,6 +744,11 @@ tasks:
     commands:
       - func: install dependencies
       - func: run atlas tests
+  - name: test-atlas-data-lake
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongohoused
+      - func: run data lake tests
   - name: test-auth-kerberos
     tags:
       - auth
@@ -1111,6 +1137,7 @@ buildvariants:
       - test-2.6-replica_set
       - test-2.6-sharded_cluster
       - test-atlas-connectivity
+      - test-atlas-data-lake
       - test-auth-kerberos
       - test-auth-ldap
       - test-tls-support
@@ -1178,6 +1205,7 @@ buildvariants:
       - test-2.6-replica_set
       - test-2.6-sharded_cluster
       - test-atlas-connectivity
+      - test-atlas-data-lake
       - test-auth-kerberos
       - test-auth-ldap
   - name: ubuntu-14.04-erbium
@@ -1215,6 +1243,7 @@ buildvariants:
       - test-3.2-replica_set
       - test-3.2-sharded_cluster
       - test-atlas-connectivity
+      - test-atlas-data-lake
       - test-auth-kerberos
       - test-auth-ldap
       - test-tls-support

.evergreen/config.yml.in

@@ -103,6 +103,19 @@ functions:
       params:
         file: mo-expansion.yml
 
+  bootstrap mongohoused:
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
+    - command: shell.exec
+      params:
+        background: true
+        script: |
+          ${PREPARE_SHELL}
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
+
   "run tests":
     - command: shell.exec
       type: test
@@ -229,8 +242,15 @@ functions:
         script: |
           MONGODB_URI='${plain_auth_mongodb_uri}' NODE_LTS_NAME='${NODE_LTS_NAME}' \
           bash ${PROJECT_DIRECTORY}/.evergreen/run-ldap-tests.sh
-
-  "run tls tests":  
+  run data lake tests:
+      - command: shell.exec
+        type: test
+        params:
+          working_dir: src
+          script: |
+            MONGODB_URI='mongodb://mhuser:pencil@localhost' NODE_LTS_NAME='${NODE_LTS_NAME}' \
+            bash ${PROJECT_DIRECTORY}/.evergreen/run-data-lake-tests.sh
+  "run tls tests":
     - command: shell.exec
       type: test
       params:
@@ -424,7 +444,7 @@ functions:
           UNIFIED=${UNIFIED} \
           CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/rsa/ca.pem" \
           OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
-          sh ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh
 
   run-valid-ocsp-server:
     - command: shell.exec

.evergreen/generate_evergreen_tasks.js

@@ -101,6 +101,14 @@ Array.prototype.push.apply(TASKS, [
       { func: 'run atlas tests' }
     ]
   },
+  {
+    name: "test-atlas-data-lake",
+    commands: [
+      { func: 'install dependencies' },
+      { func: 'bootstrap mongohoused' },
+      { func: 'run data lake tests' }
+    ]
+  },
   {
     name: 'test-auth-kerberos',
     tags: ['auth', 'kerberos'],

.evergreen/run-data-lake-tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit  # Exit the script with error if any of the commands fail
+
+export PROJECT_DIRECTORY="$(pwd)"
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
+export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+
+echo $MONGODB_URI;
+npm run check:adl

package.json

@@ -96,6 +96,7 @@
     "check:test": "mocha --recursive test/functional test/unit",
     "check:ts": "tsc -v && tsc --noEmit",
     "check:atlas": "mocha --config \"test/manual/mocharc.json\" test/manual/atlas_connectivity.test.js",
+    "check:adl": "mocha test/manual/data_lake.test.js",
     "check:ocsp": "mocha --config \"test/manual/mocharc.json\" test/manual/ocsp_support.test.js",
     "check:kerberos": "mocha --config \"test/manual/mocharc.json\" test/manual/kerberos.test.js",
     "check:tls": "mocha --config \"test/manual/mocharc.json\" test/manual/tls_support.test.js",

test/functional/spec-runner/context.js

@@ -30,8 +30,18 @@ class Thread {
 }
 
 class TestRunnerContext {
-  constructor() {
-    this.url = null;
+  constructor(opts) {
+    const defaults = {
+      password: undefined,
+      user: undefined,
+      useSessions: true,
+      skipPrepareDatabase: false
+    };
+    opts = Object.assign({}, defaults, opts || {});
+    this.skipPrepareDatabase = opts.skipPrepareDatabase;
+    this.useSessions = opts.useSessions;
+    this.user = opts.user;
+    this.password = opts.password;
     this.sharedClient = null;
     this.failPointClients = [];
     this.appliedFailPoints = [];
@@ -57,9 +67,8 @@ class TestRunnerContext {
 
   setup(config) {
     this.sharedClient = config.newClient(
-      resolveConnectionString(config, { useMultipleMongoses: true })
+      resolveConnectionString(config, { useMultipleMongoses: true }, this)
     );
-
     if (config.topologyType === 'Sharded') {
       this.failPointClients = config.options.hosts.map(proxy =>
         config.newClient(`mongodb://${proxy.host}:${proxy.port}/`)

test/functional/spec-runner/index.js

@@ -149,7 +149,6 @@ function generateTopologyTests(testSuites, testContext, filter) {
               if (spec.failPoint) {
                 testPromise = testPromise.then(() => testContext.disableFailPoint(spec.failPoint));
               }
-
               return testPromise.then(() => validateOutcome(spec, testContext));
             });
           });
@@ -165,6 +164,9 @@ function prepareDatabaseForSuite(suite, context) {
   context.collectionName = suite.collection_name || 'spec_collection';
 
   const db = context.sharedClient.db(context.dbName);
+
+  if (context.skipPrepareDatabase) return Promise.resolve();
+
   const setupPromise = db
     .admin()
     .command({ killAllSessions: [] })
@@ -279,15 +281,22 @@ function runTestSuiteTest(configuration, spec, context) {
     )
   );
 
-  const url = resolveConnectionString(configuration, spec);
+  const url = resolveConnectionString(configuration, spec, context);
   const client = configuration.newClient(url, clientOptions);
   CMAP_EVENTS.forEach(eventName => client.on(eventName, event => context.cmapEvents.push(event)));
   SDAM_EVENTS.forEach(eventName => client.on(eventName, event => context.sdamEvents.push(event)));
+  let pingTracker = 0;
   client.on('commandStarted', event => {
     if (IGNORED_COMMANDS.has(event.commandName)) {
       return;
     }
 
+    // This gets rid of the first ping command
+    if (event.commandName === 'ping') {
+      pingTracker++;
+      if (pingTracker === 1) return;
+    }
+
     context.commandEvents.push(event);
 
     // very useful for debugging
@@ -305,22 +314,24 @@ function runTestSuiteTest(configuration, spec, context) {
 
     let session0, session1;
     let savedSessionData;
-    try {
-      session0 = client.startSession(
-        Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session0))
-      );
-      session1 = client.startSession(
-        Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session1))
-      );
 
-      savedSessionData = {
-        session0: JSON.parse(EJSON.stringify(session0.id)),
-        session1: JSON.parse(EJSON.stringify(session1.id))
-      };
-    } catch (err) {
-      // ignore
-    }
+    if (context.useSessions) {
+      try {
+        session0 = client.startSession(
+          Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session0))
+        );
+        session1 = client.startSession(
+          Object.assign({}, sessionOptions, parseSessionOptions(spec.sessionOptions.session1))
+        );
 
+        savedSessionData = {
+          session0: JSON.parse(EJSON.stringify(session0.id)),
+          session1: JSON.parse(EJSON.stringify(session1.id))
+        };
+      } catch (err) {
+        // ignore
+      }
+    }
     // enable to see useful APM debug information at the time of actual test run
     // displayCommands = true;
 

test/functional/spec-runner/utils.js

@@ -1,12 +1,15 @@
 'use strict';
 
-function resolveConnectionString(configuration, spec) {
+function resolveConnectionString(configuration, spec, context) {
   const isShardedEnvironment = configuration.topologyType === 'Sharded';
   const useMultipleMongoses = spec && !!spec.useMultipleMongoses;
-
-  return isShardedEnvironment && !useMultipleMongoses
-    ? `mongodb://${configuration.host}:${configuration.port}/${configuration.db}?directConnection=false`
-    : configuration.url();
+  const user = context && context.user;
+  const password = context && context.password;
+  const connectionString =
+    isShardedEnvironment && !useMultipleMongoses
+      ? `mongodb://${configuration.host}:${configuration.port}/${configuration.db}?directConnection=false`
+      : configuration.url(user, password);
+  return connectionString;
 }
 
 module.exports = { resolveConnectionString };

test/manual/data_lake.test.js

@@ -0,0 +1,73 @@
+'use strict';
+const { expect } = require('chai');
+const path = require('path');
+const { TestRunnerContext } = require('../functional/spec-runner');
+const { gatherTestSuites } = require('../functional/spec-runner');
+const { generateTopologyTests } = require('../functional/spec-runner');
+const { withClient } = require('../functional/shared');
+
+describe('Atlas Data Lake', function () {
+  context('spec tests', function () {
+    const testContext = new TestRunnerContext({
+      skipPrepareDatabase: true,
+      useSessions: false,
+      user: 'mhuser',
+      password: 'pencil'
+    });
+
+    let testSuites = gatherTestSuites(path.resolve(__dirname, '../spec/atlas-data-lake-testing'));
+
+    after(() => testContext.teardown());
+    before(function () {
+      return testContext.setup(this.configuration);
+    });
+
+    generateTopologyTests(testSuites, testContext);
+  });
+
+  describe('prose Tests', function () {
+    it(
+      'should properly constructs and issues a killCursors command',
+      withClient('mongodb://mhuser:pencil@localhost', function (client, done) {
+        const db = client.db('admin');
+        db.command({ killCursors: 'kill_cursor_collection' }, err => {
+          expect(err).to.not.exist;
+          done();
+        });
+      })
+    );
+    it(
+      'should connect without authentication',
+      withClient('mongodb://localhost', function (client, done) {
+        expect(client).to.exist;
+        done();
+      })
+    );
+    it(
+      'should connect with auth SCRAM-SHA-1',
+      withClient('mongodb://mhuser:pencil@localhost?authMechanism=SCRAM-SHA-1', function (
+        client,
+        done
+      ) {
+        const db = client.db('admin');
+        db.command({ killCursors: 'kill_cursor_collection' }, err => {
+          expect(err).to.not.exist;
+          done();
+        });
+      })
+    );
+    it(
+      'should connect with auth SCRAM-SHA-256',
+      withClient('mongodb://mhuser:pencil@localhost?authMechanism=SCRAM-SHA-256', function (
+        client,
+        done
+      ) {
+        const db = client.db('admin');
+        db.command({ killCursors: 'kill_cursor_collection' }, err => {
+          expect(err).to.not.exist;
+          done();
+        });
+      })
+    );
+  });
+});