review comments

Author: prashantmital

.evergreen/config.yml

@@ -424,6 +424,7 @@ functions:
             COMPRESSORS=${COMPRESSORS} \
             AUTH=${AUTH} \
             SSL=${SSL} \
+            DATA_LAKE=${DATA_LAKE} \
             sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run enterprise auth tests":
@@ -446,23 +447,6 @@ functions:
           # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
           PYTHON_BINARY=${PYTHON_BINARY} ATLAS_REPL='${atlas_repl}' ATLAS_SHRD='${atlas_shrd}' ATLAS_FREE='${atlas_free}' ATLAS_TLS11='${atlas_tls11}' ATLAS_TLS12='${atlas_tls12}'  sh ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
 
-  "run datalake tests":
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: "src"
-        script: |
-          if [ -n "${set_xtrace_on}" ]; then
-            set -o xtrace
-            export SET_XTRACE_ON="${set_xtrace_on}"
-          fi
-          ${PREPARE_SHELL}
-          PYTHON_BINARY=${PYTHON_BINARY} \
-            C_EXTENSIONS=${C_EXTENSIONS} \
-            AUTH=${AUTH} \
-            SSL=${SSL} \
-            sh ${PROJECT_DIRECTORY}/.evergreen/run-datalake-tests.sh
-
   "add aws auth variables to file":
     - command: shell.exec
       type: test
@@ -1196,9 +1180,9 @@ tasks:
     - name: atlas-data-lake-tests
       commands:
         - func: "bootstrap datalake"
-        - func: "run datalake tests"
+        - func: "run tests"
           vars:
-            TOPOLOGY: "server"
+            DATA_LAKE: "true"
 
     - name: test-ocsp-rsa-valid-cert-server-staples
       tags: ["ocsp", "ocsp-rsa", "ocsp-staple"]
@@ -2593,10 +2577,10 @@ buildvariants:
 - matrix_name: "data-lake-spec-tests"
   matrix_spec:
     platform: ubuntu-16.04
-    python-version: ["2.7", "3.7", "3.8"]
+    python-version: ["2.7", "3.4", "3.8"]
     auth: "auth"
-    ssl: "nossl"
-  display_name: "Atlas Data Lake ${python-version}"
+    c-extensions: "*"
+  display_name: "Atlas Data Lake ${python-version} ${c-extensions}"
   tasks:
     - name: atlas-data-lake-tests
 

.evergreen/run-datalake-tests.sh

@@ -14,8 +14,8 @@ else
     set +x
 fi
 
+PYTHON="$PYTHON_BINARY"
 AUTH=${AUTH:-noauth}
-SSL=${SSL:-nossl}
 PYTHON_BINARY=${PYTHON_BINARY:-}
 C_EXTENSIONS=${C_EXTENSIONS:-}
 
@@ -24,29 +24,6 @@ if [ "$AUTH" != "noauth" ]; then
     export DB_PASSWORD="pencil"
 fi
 
-if [ "$SSL" != "nossl" ]; then
-    export CLIENT_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
-    export CA_PEM="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
-fi
-
-if [ -z "$PYTHON_BINARY" ]; then
-    VIRTUALENV=$(command -v virtualenv) || true
-    if [ -z "$VIRTUALENV" ]; then
-        PYTHON=$(command -v python || command -v python3) || true
-        if [ -z "$PYTHON" ]; then
-            echo "Cannot test without python or python3 installed!"
-            exit 1
-        fi
-    else
-        $VIRTUALENV pymongotestvenv
-        . pymongotestvenv/bin/activate
-        PYTHON=python
-        trap "deactivate; rm -rf pymongotestvenv" EXIT HUP
-    fi
-else
-    PYTHON="$PYTHON_BINARY"
-fi
-
 echo "Running $AUTH tests over $SSL with python $PYTHON"
 $PYTHON -c 'import sys; print(sys.version)'
 
@@ -78,4 +55,4 @@ if [ -z "$C_EXTENSIONS" -a $PYTHON_IMPL = "CPython" ]; then
 fi
 
 echo "Running tests"
-$PYTHON setup.py $C_EXTENSIONS test -s test.data_lake.test_data_lake $OUTPUT
+$PYTHON setup.py $C_EXTENSIONS test -s test.data_lake.test_data_lake $OUTPUT

.evergreen/run-tests.sh

@@ -19,7 +19,6 @@ else
     set +x
 fi
 
-
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
 PYTHON_BINARY=${PYTHON_BINARY:-}
@@ -30,6 +29,7 @@ COMPRESSORS=${COMPRESSORS:-}
 TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
 SETDEFAULTENCODING=${SETDEFAULTENCODING:-}
+DATA_LAKE=${DATA_LAKE:-}
 
 if [ -n "$COMPRESSORS" ]; then
     export COMPRESSORS=$COMPRESSORS
@@ -38,8 +38,13 @@ fi
 export JAVA_HOME=/opt/java/jdk8
 
 if [ "$AUTH" != "noauth" ]; then
-    export DB_USER="bob"
-    export DB_PASSWORD="pwd123"
+    if [ -z "$DATA_LAKE" ]; then
+        export DB_USER="bob"
+        export DB_PASSWORD="pwd123"
+    else
+        export DB_USER="mhuser"
+        export DB_PASSWORD="pencil"
+    fi
 fi
 
 if [ "$SSL" != "nossl" ]; then
@@ -149,9 +154,15 @@ fi
 
 PYTHON_IMPL=$($PYTHON -c "import platform, sys; sys.stdout.write(platform.python_implementation())")
 if [ $PYTHON_IMPL = "Jython" ]; then
-    EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
+    PYTHON_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
+else
+    PYTHON_ARGS=""
+fi
+
+if [ -z "$DATA_LAKE" ]; then
+    TEST_ARGS="-s test.test_data_lake"
 else
-    EXTRA_ARGS=""
+    TEST_ARGS=""
 fi
 
 # Don't download unittest-xml-reporting from pypi, which often fails.
@@ -200,7 +211,7 @@ if [ -z "$GREEN_FRAMEWORK" ]; then
         # causing this script to exit.
         $PYTHON -c "from bson import _cbson; from pymongo import _cmessage"
     fi
-    $COVERAGE_OR_PYTHON $EXTRA_ARGS $COVERAGE_ARGS setup.py $C_EXTENSIONS test $OUTPUT
+    $COVERAGE_OR_PYTHON $PYTHON_ARGS $COVERAGE_ARGS setup.py $C_EXTENSIONS test $TEST_ARGS $OUTPUT
 else
     # --no_ext has to come before "test" so there is no way to toggle extensions here.
     $PYTHON green_framework_test.py $GREEN_FRAMEWORK $OUTPUT

test/__init__.py

@@ -191,7 +191,7 @@ def __init__(self):
         self.sessions_enabled = False
         self.client = None
         self.conn_lock = threading.Lock()
-        self.data_lake = False
+        self.is_data_lake = False
 
         if COMPRESSORS:
             self.default_client_options["compressors"] = COMPRESSORS
@@ -236,7 +236,7 @@ def _init_client(self):
         if self.client is not None:
             build_info = self.client.admin.command('buildInfo')
             if 'dataLake' in build_info:
-                self.data_lake = True
+                self.is_data_lake = True
                 self.auth_enabled = True
                 self.client = self._connect(
                     host, port, username=db_user, password=db_pwd)
@@ -857,7 +857,7 @@ def teardown():
     if garbage:
         assert False, '\n'.join(garbage)
     c = client_context.client
-    if c and not client_context.data_lake:
+    if c and not client_context.is_data_lake:
         c.drop_database("pymongo-pooling-tests")
         c.drop_database("pymongo_test")
         c.drop_database("pymongo_test1")

test/data_lake/__init__.py

@@ -0,0 +1,60 @@
+# Copyright 2020-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test Atlas Data Lake."""
+
+import os
+import sys
+
+sys.path[0:0] = [""]
+
+from test import client_context, unittest
+from test.test_crud_v2 import TestSpec
+from test.utils import TestCreator
+
+
+# Location of JSON test specifications.
+_TEST_PATH = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), "data_lake")
+
+# Default test database and collection names.
+TEST_DB = 'test'
+TEST_COLLECTION = 'driverdata'
+
+
+class DataLakeTestSpec(TestSpec):
+    def maybe_skip_scenario(self, test):
+        # Skip these tests unless connected to data lake.
+        if not client_context.is_data_lake:
+            raise unittest.skipTest('Not connected to Atlas Data Lake')
+
+    def setup_scenario(self, scenario_def):
+        # Spec tests MUST NOT insert data/drop collection for
+        # data lake testing.
+        pass
+
+
+def create_test(scenario_def, test, name):
+    def run_scenario(self):
+        self.run_scenario(scenario_def, test)
+
+    return run_scenario
+
+
+test_creator = TestCreator(
+    create_test, DataLakeTestSpec, _TEST_PATH).create_tests()
+
+
+if __name__ == "__main__":
+    unittest.main()