Skip to content

Commit c673d8b

Browse files
prashantmitalprashantmital
authored
PYTHON-2318 Atlas Data Lake testing (mongodb#500)
1 parent 733ab25 commit c673d8b

14 files changed

+466
-38
lines changed

‎.evergreen/config.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,25 @@ functions:
301301
- key: MONGODB_STARTED
302302
value: "1"
303303

304+
"bootstrap data lake":
305+
- command: shell.exec
306+
type: setup
307+
params:
308+
script: |
309+
set -o xtrace
310+
${PREPARE_SHELL}
311+
cd ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake
312+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh build-mongohouse-local.sh
313+
- command: shell.exec
314+
type: setup
315+
params:
316+
background: true
317+
script: |
318+
set -o xtrace
319+
${PREPARE_SHELL}
320+
cd ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake
321+
DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh run-mongohouse-local.sh
322+
304323
"stop mongo-orchestration":
305324
- command: shell.exec
306325
params:
@@ -405,6 +424,7 @@ functions:
405424
COMPRESSORS=${COMPRESSORS} \
406425
AUTH=${AUTH} \
407426
SSL=${SSL} \
427+
DATA_LAKE=${DATA_LAKE} \
408428
sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
409429
410430
"run enterprise auth tests":
@@ -1157,6 +1177,13 @@ tasks:
11571177
commands:
11581178
- func: "run atlas tests"
11591179

1180+
- name: atlas-data-lake-tests
1181+
commands:
1182+
- func: "bootstrap data lake"
1183+
- func: "run tests"
1184+
vars:
1185+
DATA_LAKE: "true"
1186+
11601187
- name: test-ocsp-rsa-valid-cert-server-staples
11611188
tags: ["ocsp", "ocsp-rsa", "ocsp-staple"]
11621189
commands:
@@ -2547,6 +2574,16 @@ buildvariants:
25472574
tasks:
25482575
- name: "atlas-connect"
25492576

2577+
- matrix_name: "data-lake-spec-tests"
2578+
matrix_spec:
2579+
platform: ubuntu-16.04
2580+
python-version: ["2.7", "3.4", "3.8"]
2581+
auth: "auth"
2582+
c-extensions: "*"
2583+
display_name: "Atlas Data Lake ${python-version} ${c-extensions}"
2584+
tasks:
2585+
- name: atlas-data-lake-tests
2586+
25502587
- matrix_name: "ocsp-test"
25512588
matrix_spec:
25522589
platform: ubuntu-16.04

‎.evergreen/run-tests.sh

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ else
1919
set +x
2020
fi
2121

22-
2322
AUTH=${AUTH:-noauth}
2423
SSL=${SSL:-nossl}
2524
PYTHON_BINARY=${PYTHON_BINARY:-}
@@ -30,6 +29,7 @@ COMPRESSORS=${COMPRESSORS:-}
3029
TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
3130
LIBMONGOCRYPT_URL=${LIBMONGOCRYPT_URL:-}
3231
SETDEFAULTENCODING=${SETDEFAULTENCODING:-}
32+
DATA_LAKE=${DATA_LAKE:-}
3333

3434
if [ -n "$COMPRESSORS" ]; then
3535
export COMPRESSORS=$COMPRESSORS
@@ -38,8 +38,13 @@ fi
3838
export JAVA_HOME=/opt/java/jdk8
3939

4040
if [ "$AUTH" != "noauth" ]; then
41-
export DB_USER="bob"
42-
export DB_PASSWORD="pwd123"
41+
if [ -z "$DATA_LAKE" ]; then
42+
export DB_USER="bob"
43+
export DB_PASSWORD="pwd123"
44+
else
45+
export DB_USER="mhuser"
46+
export DB_PASSWORD="pencil"
47+
fi
4348
fi
4449

4550
if [ "$SSL" != "nossl" ]; then
@@ -149,9 +154,15 @@ fi
149154

150155
PYTHON_IMPL=$($PYTHON -c "import platform, sys; sys.stdout.write(platform.python_implementation())")
151156
if [ $PYTHON_IMPL = "Jython" ]; then
152-
EXTRA_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
157+
PYTHON_ARGS="-J-XX:-UseGCOverheadLimit -J-Xmx4096m"
158+
else
159+
PYTHON_ARGS=""
160+
fi
161+
162+
if [ -z "$DATA_LAKE" ]; then
163+
TEST_ARGS=""
153164
else
154-
EXTRA_ARGS=""
165+
TEST_ARGS="-s test.test_data_lake"
155166
fi
156167

157168
# Don't download unittest-xml-reporting from pypi, which often fails.
@@ -200,7 +211,7 @@ if [ -z "$GREEN_FRAMEWORK" ]; then
200211
# causing this script to exit.
201212
$PYTHON -c "from bson import _cbson; from pymongo import _cmessage"
202213
fi
203-
$COVERAGE_OR_PYTHON $EXTRA_ARGS $COVERAGE_ARGS setup.py $C_EXTENSIONS test $OUTPUT
214+
$COVERAGE_OR_PYTHON $PYTHON_ARGS $COVERAGE_ARGS setup.py $C_EXTENSIONS test $TEST_ARGS $OUTPUT
204215
else
205216
# --no_ext has to come before "test" so there is no way to toggle extensions here.
206217
$PYTHON green_framework_test.py $GREEN_FRAMEWORK $OUTPUT

‎test/__init__.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ def __init__(self):
191191
self.sessions_enabled = False
192192
self.client = None
193193
self.conn_lock = threading.Lock()
194+
self.is_data_lake = False
194195

195196
if COMPRESSORS:
196197
self.default_client_options["compressors"] = COMPRESSORS
@@ -231,6 +232,19 @@ def _connect(self, host, port, **kwargs):
231232

232233
def _init_client(self):
233234
self.client = self._connect(host, port)
235+
236+
if self.client is not None:
237+
# Return early when connected to dataLake as mongohoused does not
238+
# support the getCmdLineOpts command and is tested without TLS.
239+
build_info = self.client.admin.command('buildInfo')
240+
if 'dataLake' in build_info:
241+
self.is_data_lake = True
242+
self.auth_enabled = True
243+
self.client = self._connect(
244+
host, port, username=db_user, password=db_pwd)
245+
self.connected = True
246+
return
247+
234248
if HAVE_SSL and not self.client:
235249
# Is MongoDB configured for SSL?
236250
self.client = self._connect(host, port, **TLS_OPTIONS)
@@ -845,14 +859,14 @@ def teardown():
845859
if garbage:
846860
assert False, '\n'.join(garbage)
847861
c = client_context.client
848-
if c:
862+
if c and not client_context.is_data_lake:
849863
c.drop_database("pymongo-pooling-tests")
850864
c.drop_database("pymongo_test")
851865
c.drop_database("pymongo_test1")
852866
c.drop_database("pymongo_test2")
853867
c.drop_database("pymongo_test_mike")
854868
c.drop_database("pymongo_test_bernie")
855-
c.close()
869+
c.close()
856870

857871
# Jython does not support gc.get_objects.
858872
if not sys.platform.startswith('java'):

‎test/crud_v2_format.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Copyright 2020-present MongoDB, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""v2 format CRUD test runner.
16+
17+
https://github.com/mongodb/specifications/blob/master/source/crud/tests/README.rst
18+
"""
19+
20+
from test.utils_spec_runner import SpecRunner
21+
22+
23+
class TestCrudV2(SpecRunner):
24+
# Default test database and collection names.
25+
TEST_DB = None
26+
TEST_COLLECTION = None
27+
28+
def get_scenario_db_name(self, scenario_def):
29+
"""Crud spec says database_name is optional."""
30+
return scenario_def.get('database_name', self.TEST_DB)
31+
32+
def get_scenario_coll_name(self, scenario_def):
33+
"""Crud spec says collection_name is optional."""
34+
return scenario_def.get('collection_name', self.TEST_COLLECTION)
35+
36+
def get_object_name(self, op):
37+
"""Crud spec says object is optional and defaults to 'collection'."""
38+
return op.get('object', 'collection')
39+
40+
def get_outcome_coll_name(self, outcome, collection):
41+
"""Crud spec says outcome has an optional 'collection.name'."""
42+
return outcome['collection'].get('name', collection.name)
43+
44+
def setup_scenario(self, scenario_def):
45+
"""Allow specs to override a test's setup."""
46+
# PYTHON-1935 Only create the collection if there is data to insert.
47+
if scenario_def['data']:
48+
super(TestCrudV2, self).setup_scenario(scenario_def)

‎test/data_lake/aggregate.json

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"collection_name": "driverdata",
3+
"database_name": "test",
4+
"tests": [
5+
{
6+
"description": "Aggregate with pipeline (project, sort, limit)",
7+
"operations": [
8+
{
9+
"object": "collection",
10+
"name": "aggregate",
11+
"arguments": {
12+
"pipeline": [
13+
{
14+
"$project": {
15+
"_id": 0
16+
}
17+
},
18+
{
19+
"$sort": {
20+
"a": 1
21+
}
22+
},
23+
{
24+
"$limit": 2
25+
}
26+
]
27+
},
28+
"result": [
29+
{
30+
"a": 1,
31+
"b": 2,
32+
"c": 3
33+
},
34+
{
35+
"a": 2,
36+
"b": 3,
37+
"c": 4
38+
}
39+
]
40+
}
41+
],
42+
"expectations": [
43+
{
44+
"command_started_event": {
45+
"command": {
46+
"aggregate": "driverdata"
47+
}
48+
}
49+
}
50+
]
51+
}
52+
]
53+
}

‎test/data_lake/estimatedDocumentCount.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"collection_name": "driverdata",
3+
"database_name": "test",
4+
"tests": [
5+
{
6+
"description": "estimatedDocumentCount succeeds",
7+
"operations": [
8+
{
9+
"object": "collection",
10+
"name": "estimatedDocumentCount",
11+
"result": 15
12+
}
13+
],
14+
"expectations": [
15+
{
16+
"command_started_event": {
17+
"command": {
18+
"count": "driverdata"
19+
}
20+
}
21+
}
22+
]
23+
}
24+
]
25+
}

‎test/data_lake/find.json

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"collection_name": "driverdata",
3+
"database_name": "test",
4+
"tests": [
5+
{
6+
"description": "Find with projection and sort",
7+
"operations": [
8+
{
9+
"object": "collection",
10+
"name": "find",
11+
"arguments": {
12+
"filter": {
13+
"b": {
14+
"$gt": 5
15+
}
16+
},
17+
"projection": {
18+
"_id": 0
19+
},
20+
"sort": {
21+
"a": 1
22+
},
23+
"limit": 5
24+
},
25+
"result": [
26+
{
27+
"a": 5,
28+
"b": 6,
29+
"c": 7
30+
},
31+
{
32+
"a": 6,
33+
"b": 7,
34+
"c": 8
35+
},
36+
{
37+
"a": 7,
38+
"b": 8,
39+
"c": 9
40+
},
41+
{
42+
"a": 8,
43+
"b": 9,
44+
"c": 10
45+
},
46+
{
47+
"a": 9,
48+
"b": 10,
49+
"c": 11
50+
}
51+
]
52+
}
53+
],
54+
"expectations": [
55+
{
56+
"command_started_event": {
57+
"command": {
58+
"find": "driverdata"
59+
}
60+
}
61+
}
62+
]
63+
}
64+
]
65+
}

0 commit comments

Comments
 (0)