chore(ci): adopt own toolchain MONGOSH-864 (#2401)

Author: addaleax

.evergreen.yml

@@ -122,8 +122,10 @@ functions:
           DISTRO_ID: ${distro_id}
           MONOGDB_DRIVER_VERSION_OVERRIDE: ${mongodb_driver_version_override}
         script: |
-          source .evergreen/install-node.sh
-          source .evergreen/install-npm-deps.sh
+          set -e
+          set -x
+          .evergreen/install-node.sh
+          .evergreen/install-npm-deps.sh
     - command: shell.exec
       params:
         working_dir: src
@@ -160,8 +162,10 @@ functions:
           DISTRO_ID: ${distro_id}
           MONOGDB_DRIVER_VERSION_OVERRIDE: ${mongodb_driver_version_override}
         script: |
-          source .evergreen/install-node.sh
-          source .evergreen/install-npm-deps.sh
+          set -e
+          set -x
+          .evergreen/install-node.sh
+          .evergreen/install-npm-deps.sh
     - command: s3.get
       type: setup
       params:

.evergreen/compile-artifact.sh

@@ -71,7 +71,9 @@ elif [ -n "$MONGOSH_SHARED_OPENSSL" ]; then
     "--shared-openssl-libname='"$MONGOSH_OPENSSL_LIBNAME"'",
     "--shared-zlib"
   ]'
-  export LD_LIBRARY_PATH=/tmp/m/opt/lib
+  # python3's ssl module may not work with the OpenSSL we built here,
+  # so prefix the devtools toolchain one
+  export LD_LIBRARY_PATH=/opt/devtools/lib:/tmp/m/opt/lib
 fi
 
 export PUPPETEER_SKIP_DOWNLOAD="true"

.evergreen/evergreen.yml.in

@@ -131,8 +131,10 @@ functions:
           DISTRO_ID: ${distro_id}
           MONOGDB_DRIVER_VERSION_OVERRIDE: ${mongodb_driver_version_override}
         script: |
-          source .evergreen/install-node.sh
-          source .evergreen/install-npm-deps.sh
+          set -e
+          set -x
+          .evergreen/install-node.sh
+          .evergreen/install-npm-deps.sh
     - command: shell.exec
       params:
         working_dir: src
@@ -169,8 +171,10 @@ functions:
           DISTRO_ID: ${distro_id}
           MONOGDB_DRIVER_VERSION_OVERRIDE: ${mongodb_driver_version_override}
         script: |
-          source .evergreen/install-node.sh
-          source .evergreen/install-npm-deps.sh
+          set -e
+          set -x
+          .evergreen/install-node.sh
+          .evergreen/install-npm-deps.sh
     - command: s3.get
       type: setup
       params:

.evergreen/install-node-source.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 set -e
 set -x
 export BASEDIR="$PWD/.evergreen"
@@ -12,63 +13,22 @@ if [ "$OS" == "Windows_NT" ]; then
 
   curl -sSfLO https://raw.githubusercontent.com/mongodb-js/compass/42e6142ae08be6fec944b80ff6289e6bcd11badf/.evergreen/node-gyp-bug-workaround.sh && bash node-gyp-bug-workaround.sh
 else
-  if which realpath; then # No realpath on macOS, but also not needed there
-    export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink
-  fi
-  # Some Node.js driver versions leave a ~/.npmrc file lying around
-  # that breaks nvm because it contains a 'prefix=' option (pointing
-  # to a directory that no longer exists anyway).
-  if [ -e "$HOME/.npmrc" ]; then
-    # different `sed` arguments on macOS than for GNU sed ...
-    if [ `uname` == Darwin ]; then
-      sed -i'~' -e 's/^prefix=.*$//' "$HOME/.npmrc"
-    else
-      sed -i "$HOME/.npmrc" -e 's/^prefix=.*$//'
-    fi
-  fi
-  export NVM_DIR="$BASEDIR/.nvm"
-  mkdir -p "${NVM_DIR}"
-
-  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
-
-  echo "Setting NVM environment home: $NVM_DIR"
-  [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-  [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
-
-  set +x # nvm is very verbose
-
-  # A few distros where pre-built node20 does not work out of the box and hence
-  # needs to be built from source
-  if [[ "${DISTRO_ID}" =~ ^(amazon2-|rhel7|ubuntu18|suse12) ]] && [[ "$NODE_JS_VERSION" =~ ^20 ]];
-  then
-    NODE_JS_SOURCE_VERSION="$NODE_JS_VERSION"
-    if echo $NODE_JS_VERSION | grep -q ^20 ; then
-      # Node.js 20.11.1 is the last 20.x that builds out of the box on RHEL7
-      # https://github.com/nodejs/node/issues/52223
-      NODE_JS_SOURCE_VERSION=20.11.1
-    fi
-    env NODE_JS_VERSION="$NODE_JS_SOURCE_VERSION" bash "$BASEDIR/install-node-source.sh"
-    nvm use $NODE_JS_SOURCE_VERSION
-  else
+  if [ `uname` = Darwin ]; then
+    export NVM_DIR="$BASEDIR/.nvm"
+    mkdir -p "${NVM_DIR}"
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+
+    echo "Setting NVM environment home: $NVM_DIR"
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+    [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"
+    set +x # nvm is very verbose
     echo nvm install --no-progress $NODE_JS_VERSION && nvm alias default $NODE_JS_VERSION
     nvm install --no-progress $NODE_JS_VERSION
     nvm alias default $NODE_JS_VERSION
     nvm use $NODE_JS_VERSION
+    set -x
   fi
-  set -x
-
-  if env PATH="/opt/chefdk/gitbin:$PATH" git --version | grep -q 'git version 1.'; then
-    (cd "$BASEDIR" &&
-      curl -sSfL https://github.com/git/git/archive/refs/tags/v2.31.1.tar.gz | tar -xvz &&
-      mv git-2.31.1 git-2 &&
-      cd git-2 &&
-      make -j8 NO_EXPAT=1)
-  fi
-
   npm cache clear --force || true # Try to work around `Cannot read property 'pickAlgorithm' of null` errors in CI
-  # Started observing CI failures on RHEL 7.2 (s390x) for installing npm, all
-  # related to network issues hence adding a retry with backoff here.
-  bash "$BASEDIR/retry-with-backoff.sh" npm i -g npm@$NPM_VERSION
 fi
 
 . "$BASEDIR/setup-env.sh"

.evergreen/install-node.sh

@@ -1,10 +1,9 @@
+#!/bin/bash
 set -e
 set -x
+export BASEDIR="$PWD/.evergreen"
 
-if [[ "${DISTRO_ID}" =~ ^(rhel|ubuntu1804) ]]; then
-  # RHEL and Ubuntu 18.04 use Python 3.6 which isn't supported by newer node-gyp versions
-  npm i node-gyp@9 --verbose --force
-fi
+. "$BASEDIR/setup-env.sh"
 
 npm ci --verbose
 echo "MONOGDB_DRIVER_VERSION_OVERRIDE:$MONOGDB_DRIVER_VERSION_OVERRIDE"

.evergreen/install-npm-deps.sh

@@ -2,16 +2,9 @@ set -e
 set -x
 
 OS_ARCH="$(uname "-m")"
-if [ "$OS_ARCH" = "ppc64le" ] || [ "$OS_ARCH" = "ppc64" ]; then
-    echo "[INFO] Choosing v4 because OS_ARCH is $OS_ARCH"
-    export TOOLCHAIN_PATH='/opt/mongodbtoolchain/v4/bin'
-else
-    echo "[INFO] Choosing v3 because OS_ARCH is $OS_ARCH"
-    export TOOLCHAIN_PATH='/opt/mongodbtoolchain/v3/bin'
-fi
 
 export BASEDIR="$PWD/.evergreen"
-export PATH="/cygdrive/c/python/Python311/Scripts:/cygdrive/c/python/Python311:/cygdrive/c/Python311/Scripts:/cygdrive/c/Python311:/opt/python/3.6/bin:$BASEDIR/mingit/cmd:$BASEDIR/mingit/mingw64/libexec/git-core:$BASEDIR/git-2:$BASEDIR/npm-10/node_modules/.bin:$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/java/jdk16/bin:/opt/chefdk/gitbin:/cygdrive/c/cmake/bin:$TOOLCHAIN_PATH:$PATH"
+export PATH="$BASEDIR/npm-10/node_modules/.bin:$BASEDIR/node-v$NODE_JS_VERSION-win-x64:/opt/java/jdk16/bin:$PATH"
 
 export MONGOSH_GLOBAL_CONFIG_FILE_FOR_TESTING="$BASEDIR/../../testing/tests-globalconfig.conf"
 
@@ -26,38 +19,46 @@ fi
 echo "TERM variable is set to '${TERM:-}'"
 
 if [ "$OS" != "Windows_NT" ]; then
-  if which realpath; then # No realpath on macOS, but also not needed there
-    export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink
-  fi
-  export NVM_DIR="$BASEDIR/.nvm"
-  echo "Setting NVM environment home: $NVM_DIR"
-  [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-  set +x # nvm is very verbose
-  echo nvm use $NODE_JS_VERSION || nvm use 20.11.1
-  nvm use $NODE_JS_VERSION || nvm use 20.11.1 # see install-node.sh
-  set -x
-  export PATH="$NVM_BIN:$PATH"
-
   if [ `uname` = Darwin ]; then
     echo "Using clang version:"
     (which clang && clang --version)
 
     echo "Using clang++ version:"
     (which clang++ && clang++ --version)
+
+    export NVM_DIR="$BASEDIR/.nvm"
+    echo "Setting NVM environment home: $NVM_DIR"
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+    set +x # nvm is very verbose
+    nvm use $NODE_JS_VERSION
+    set -x
+    export PATH="$NVM_BIN:$PATH"
   else
+    export PATH="/opt/devtools/bin:$PATH"
+    export GIT_EXEC_PATH="/opt/devtools/libexec/git-core"
     export CC=gcc
     export CXX=g++
+    export PYTHON="/opt/devtools/bin/python3"
 
     echo "Using gcc version:"
     (which gcc && gcc --version)
 
     echo "Using g++ version:"
     (which g++ && g++ --version)
   fi
+else
+  export PATH="/cygdrive/c/python/Python311/Scripts:/cygdrive/c/python/Python311:/cygdrive/c/Python311/Scripts:/cygdrive/c/Python311:/cygdrive/c/cmake/bin:$PATH"
+fi
 
-  if [ -x "$BASEDIR/git-2/git" ]; then
-    export GIT_EXEC_PATH="$BASEDIR/git-2"
-  fi
+NODE_JS_MAJOR_VERSION=$(echo "$NODE_JS_VERSION" | awk -F . '{print $1}')
+if echo "$NODE_JS_MAJOR_VERSION" | grep -q '^[0-9]*$'; then
+  export PATH="/opt/devtools/node$NODE_JS_MAJOR_VERSION/bin:$PATH"
+  echo "Detected Node.js version (requested v${NODE_JS_MAJOR_VERSION}.x):"
+  node -v
+  node -v | grep -q "^v$NODE_JS_MAJOR_VERSION"
+else
+  echo "Cannot identify major version from NODE_JS_VERSION: $NODE_JS_VERSION"
+  exit 1
 fi
 
 export EVERGREEN_EXPANSIONS_PATH="$BASEDIR/../../tmp/expansions.yaml"

.evergreen/setup-env.sh

@@ -47,6 +47,7 @@ function try_connect_ipv4only_dualstackhostname() {
   # Use a second docker container to be able to modify /etc/hosts easily
   cat <<EOF | docker run -i --rm --network host -v /:/host "${DOCKER_BASE_IMG}" bash && FAILED=no || FAILED=yes
 export PATH=/host$(echo "$PATH" | sed 's~:~:/host~g'):\$PATH
+if [ -e /host/opt/devtools/lib ]; then cp -v /host/opt/devtools/lib/lib{crypto,ssl}* /lib/x86_64-linux-gnu ; fi # Node.js on the host may require modern OpenSSL
 apt update && apt -y install openssl # necessary on Ubuntu 20.04
 
 set -e