Skip to content

Commit 54f5c2d

Browse files
nbbeekennbbeeken
authored
fix(NODE-3528): add support for snappy v7 (#2947)
1 parent cc0cc20 commit 54f5c2d

File tree

9 files changed

+6739
-92
lines changed

9 files changed

+6739
-92
lines changed

‎.evergreen/config.yml

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,17 @@ functions:
480480
rm -f ./prepare_client_encryption.sh
481481
482482
UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
483+
run custom snappy tests:
484+
- command: subprocess.exec
485+
params:
486+
working_dir: src
487+
timeout_secs: 60
488+
env:
489+
MONGODB_URI: '${MONGODB_URI}'
490+
PROJECT_DIRECTORY: '${PROJECT_DIRECTORY}'
491+
binary: bash
492+
args:
493+
- '${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh'
483494
upload test results:
484495
- command: attach.xunit_results
485496
params:
@@ -1545,16 +1556,34 @@ tasks:
15451556
- func: run checks
15461557
- name: run-custom-csfle-tests
15471558
tags:
1548-
- run-custom-csfle-tests
1559+
- run-custom-dependency-tests
15491560
commands:
15501561
- func: install dependencies
15511562
vars:
15521563
NODE_LTS_NAME: erbium
15531564
- func: bootstrap mongo-orchestration
15541565
vars:
1555-
VERSION: '4.4'
1566+
VERSION: '5.0'
15561567
TOPOLOGY: server
15571568
- func: run custom csfle tests
1569+
vars:
1570+
UNIFIED: 1
1571+
NODE_LTS_NAME: erbium
1572+
- name: run-custom-snappy-tests
1573+
tags:
1574+
- run-custom-dependency-tests
1575+
commands:
1576+
- func: install dependencies
1577+
vars:
1578+
NODE_LTS_NAME: erbium
1579+
- func: bootstrap mongo-orchestration
1580+
vars:
1581+
VERSION: '5.0'
1582+
TOPOLOGY: server
1583+
- func: run custom snappy tests
1584+
vars:
1585+
UNIFIED: 1
1586+
NODE_LTS_NAME: erbium
15581587
buildvariants:
15591588
- name: macos-1014-fermium
15601589
display_name: macOS 10.14 Node Fermium
@@ -2109,11 +2138,12 @@ buildvariants:
21092138
run_on: rhel70
21102139
tasks:
21112140
- run-checks
2112-
- name: ubuntu1804-custom-csfle-tests
2113-
display_name: Custom FLE Version Test
2114-
run_on: ubuntu1804-test
2141+
- name: ubuntu1804-custom-dependency-tests
2142+
display_name: Custom Dependency Version Test
2143+
run_on: ubuntu1804-large
21152144
tasks:
21162145
- run-custom-csfle-tests
2146+
- run-custom-snappy-tests
21172147
- name: ubuntu1804-test-mongodb-aws
21182148
display_name: MONGODB-AWS Auth test
21192149
run_on: ubuntu1804-test

‎.evergreen/config.yml.in

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,18 @@ functions:
522522

523523
UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
524524

525+
"run custom snappy tests":
526+
- command: subprocess.exec
527+
params:
528+
working_dir: "src"
529+
timeout_secs: 60
530+
env:
531+
MONGODB_URI: ${MONGODB_URI}
532+
PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
533+
binary: bash
534+
args:
535+
- "${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh"
536+
525537
"upload test results":
526538
- command: attach.xunit_results
527539
params:

‎.evergreen/generate_evergreen_tasks.js

Lines changed: 61 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,7 @@ const OPERATING_SYSTEMS = [
7474
)
7575
);
7676

77-
const WINDOWS_SKIP_TAGS = new Set([
78-
'atlas-connect',
79-
'auth'
80-
]);
77+
const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth']);
8178

8279
const BASE_TASKS = [];
8380
const TASKS = [];
@@ -109,9 +106,7 @@ function makeTask({ mongoVersion, topology }) {
109106
}
110107

111108
MONGODB_VERSIONS.forEach(mongoVersion => {
112-
TOPOLOGIES.forEach(topology =>
113-
BASE_TASKS.push(makeTask({ mongoVersion, topology }))
114-
);
109+
TOPOLOGIES.forEach(topology => BASE_TASKS.push(makeTask({ mongoVersion, topology })));
115110
});
116111

117112
TASKS.push(
@@ -125,7 +120,8 @@ TASKS.push(
125120
tags: ['auth', 'kerberos', 'legacy'],
126121
commands: [
127122
{ func: 'install dependencies' },
128-
{ func: 'run kerberos tests',
123+
{
124+
func: 'run kerberos tests',
129125
vars: {
130126
UNIFIED: 0
131127
}
@@ -137,7 +133,8 @@ TASKS.push(
137133
tags: ['auth', 'kerberos', 'unified'],
138134
commands: [
139135
{ func: 'install dependencies' },
140-
{ func: 'run kerberos tests',
136+
{
137+
func: 'run kerberos tests',
141138
vars: {
142139
UNIFIED: 1
143140
}
@@ -296,7 +293,7 @@ OCSP_VERSIONS.forEach(VERSION => {
296293
const AWS_AUTH_TASKS = [];
297294

298295
AWS_AUTH_VERSIONS.forEach(VERSION => {
299-
const name = (ex) => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
296+
const name = ex => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
300297
// AWS_AUTH_TASKS.push(name);
301298

302299
const aws_funcs = [
@@ -325,10 +322,10 @@ AWS_AUTH_VERSIONS.forEach(VERSION => {
325322
{ func: 'setup aws env' },
326323
fn
327324
]
328-
}))
325+
}));
329326

330327
TASKS.push(...aws_tasks);
331-
AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name))
328+
AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name));
332329
});
333330

334331
const BUILD_VARIANTS = [];
@@ -342,31 +339,33 @@ const getTaskList = (() => {
342339
return memo[key];
343340
}
344341
const taskList = BASE_TASKS.concat(TASKS);
345-
const ret = taskList.filter(task => {
346-
if (task.name.match(/^aws/)) return false;
342+
const ret = taskList
343+
.filter(task => {
344+
if (task.name.match(/^aws/)) return false;
347345

348-
// skip unsupported tasks on windows
349-
if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
350-
return false;
351-
}
346+
// skip unsupported tasks on windows
347+
if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
348+
return false;
349+
}
352350

353-
const tasksWithVars = task.commands.filter(task => !!task.vars);
354-
if (!tasksWithVars.length) {
355-
return true;
356-
}
351+
const tasksWithVars = task.commands.filter(task => !!task.vars);
352+
if (!tasksWithVars.length) {
353+
return true;
354+
}
357355

358-
// kerberos tests don't require mongo orchestration
359-
if (task.tags.filter(tag => tag === 'kerberos').length) {
360-
return true;
361-
}
356+
// kerberos tests don't require mongo orchestration
357+
if (task.tags.filter(tag => tag === 'kerberos').length) {
358+
return true;
359+
}
362360

363-
const { VERSION } = tasksWithVars[0].vars || {};
364-
if (VERSION === 'latest') {
365-
return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
366-
}
361+
const { VERSION } = tasksWithVars[0].vars || {};
362+
if (VERSION === 'latest') {
363+
return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
364+
}
367365

368-
return semver.satisfies(semver.coerce(VERSION), mongoVersion);
369-
}).map(x => x.name);
366+
return semver.satisfies(semver.coerce(VERSION), mongoVersion);
367+
})
368+
.map(x => x.name);
370369

371370
memo[key] = ret;
372371
return ret;
@@ -419,38 +418,48 @@ SINGLETON_TASKS.push({
419418
]
420419
});
421420

422-
SINGLETON_TASKS.push({
423-
name: 'run-custom-csfle-tests',
424-
tags: ['run-custom-csfle-tests'],
421+
const oneOffFuncs = [
422+
{ func: 'run custom csfle tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } },
423+
{ func: 'run custom snappy tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } }
424+
];
425+
426+
const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({
427+
name: `${oneOffFunc.func.split(' ').join('-')}`,
428+
tags: ['run-custom-dependency-tests'],
425429
commands: [
426430
{
427431
func: 'install dependencies',
428432
vars: {
429-
NODE_LTS_NAME: 'erbium',
430-
},
433+
NODE_LTS_NAME: 'erbium'
434+
}
431435
},
432436
{
433437
func: 'bootstrap mongo-orchestration',
434438
vars: {
435-
VERSION: '4.4',
439+
VERSION: '5.0',
436440
TOPOLOGY: 'server'
437441
}
438442
},
439-
{ func: 'run custom csfle tests' }
443+
oneOffFunc
440444
]
441-
});
445+
}));
442446

443-
BUILD_VARIANTS.push({
444-
name: 'lint',
445-
display_name: 'lint',
446-
run_on: 'rhel70',
447-
tasks: ['run-checks']
448-
}, {
449-
name: 'ubuntu1804-custom-csfle-tests',
450-
display_name: 'Custom FLE Version Test',
451-
run_on: 'ubuntu1804-test',
452-
tasks: ['run-custom-csfle-tests']
453-
});
447+
SINGLETON_TASKS.push(...oneOffFuncAsTasks);
448+
449+
BUILD_VARIANTS.push(
450+
{
451+
name: 'lint',
452+
display_name: 'lint',
453+
run_on: 'rhel70',
454+
tasks: ['run-checks']
455+
},
456+
{
457+
name: 'ubuntu1804-custom-dependency-tests',
458+
display_name: 'Custom Dependency Version Test',
459+
run_on: 'ubuntu1804-large',
460+
tasks: oneOffFuncAsTasks.map(({ name }) => name)
461+
}
462+
);
454463

455464
// special case for MONGODB-AWS authentication
456465
BUILD_VARIANTS.push({
@@ -464,7 +473,7 @@ BUILD_VARIANTS.push({
464473
});
465474

466475
const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
467-
fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
476+
fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS, TASKS, SINGLETON_TASKS);
468477
fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);
469478

470479
fs.writeFileSync(`${__dirname}/config.yml`, yaml.safeDump(fileData, { lineWidth: 120 }), 'utf8');

‎.evergreen/init-nvm.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#! /usr/bin/env bash
2+
3+
export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
4+
NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
5+
export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
6+
7+
if [[ "$OS" == "Windows_NT" ]]; then
8+
NVM_HOME=$(cygpath -w "$NVM_DIR")
9+
export NVM_HOME
10+
NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
11+
export NVM_SYMLINK
12+
NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
13+
export NVM_ARTIFACTS_PATH
14+
PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
15+
export PATH
16+
echo "updated path on windows PATH=$PATH"
17+
else
18+
[ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
19+
fi

‎.evergreen/run-snappy-version-test.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#! /usr/bin/env bash
2+
3+
source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
4+
export MONGODB_URI="${MONGODB_URI}"
5+
6+
npm i --no-save snappy@6
7+
8+
npx mocha test/unit/snappy.test.js
9+
10+
npm i --no-save snappy@7
11+
12+
npx mocha test/unit/snappy.test.js

‎lib/core/connection/utils.js

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
'use strict';
22

3+
const parsePackageVersion = require('../../utils').parsePackageVersion;
4+
const MongoError = require('../error').MongoError;
5+
36
const require_optional = require('optional-require')(require);
47

58
function debugOptions(debugFields, options) {
@@ -16,7 +19,15 @@ function retrieveBSON() {
1619
BSON.native = false;
1720

1821
const optionalBSON = require_optional('bson-ext');
22+
const bsonExtVersion = parsePackageVersion(
23+
require_optional('bson-ext/package.json') || { version: '0.0.0' }
24+
);
1925
if (optionalBSON) {
26+
if (bsonExtVersion.major >= 4) {
27+
throw new MongoError(
28+
'bson-ext version 4 and above does not work with the 3.x version of the mongodb driver'
29+
);
30+
}
2031
optionalBSON.native = true;
2132
return optionalBSON;
2233
}
@@ -31,21 +42,43 @@ function noSnappyWarning() {
3142
);
3243
}
3344

45+
const PKG_VERSION = Symbol('kPkgVersion');
46+
3447
// Facilitate loading Snappy optionally
3548
function retrieveSnappy() {
36-
let snappy = require_optional('snappy');
49+
const snappy = require_optional('snappy');
3750
if (!snappy) {
38-
snappy = {
51+
return {
3952
compress: noSnappyWarning,
4053
uncompress: noSnappyWarning,
4154
compressSync: noSnappyWarning,
4255
uncompressSync: noSnappyWarning
4356
};
4457
}
58+
59+
const snappyPkg = require_optional('snappy/package.json') || { version: '0.0.0' };
60+
const version = parsePackageVersion(snappyPkg);
61+
snappy[PKG_VERSION] = version;
62+
if (version.major >= 7) {
63+
const compressOriginal = snappy.compress;
64+
const uncompressOriginal = snappy.uncompress;
65+
snappy.compress = (data, callback) => {
66+
compressOriginal(data)
67+
.then(res => callback(undefined, res))
68+
.catch(error => callback(error));
69+
};
70+
snappy.uncompress = (data, callback) => {
71+
uncompressOriginal(data)
72+
.then(res => callback(undefined, res))
73+
.catch(error => callback(error));
74+
};
75+
}
76+
4577
return snappy;
4678
}
4779

4880
module.exports = {
81+
PKG_VERSION,
4982
debugOptions,
5083
retrieveBSON,
5184
retrieveSnappy

0 commit comments

Comments
 (0)