CDRIVER-3863 add Evergreen task that builds with the new toolchain

Author: rcsanchez97

.evergreen/build-and-test-with-toolchain.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+echo "BUILDING WITH TOOLCHAIN"
+
+# Configure environment with toolchain components
+[ -d /opt/mongo-c-toolchain ] && sudo rm -r /opt/mongo-c-toolchain
+sudo mkdir /opt/mongo-c-toolchain
+TOOLCHAIN_TAR_GZ=$(readlink -f ../mongo-c-toolchain.tar.gz)
+sudo tar -xvf "${TOOLCHAIN_TAR_GZ}" -C /opt/mongo-c-toolchain
+echo "--- TOOLCHAIN MANIFEST ---"
+cat /opt/mongo-c-toolchain/MANIFEST.txt
+OLD_PATH=${PATH}
+ADDL_PATH=$(readlink -f /opt/mongo-c-toolchain/bin):${PATH}
+export CMAKE=$(readlink -f /opt/mongo-c-toolchain/bin/cmake)
+[ -x "${CMAKE}" ] || (echo "CMake (${CMAKE}) does not exist or is not executable"; exit 1)
+TOOLCHAIN_BASE_DIR=$(readlink -f /opt/mongo-c-toolchain)
+TOOLCHAIN_LIB_DIR=${TOOLCHAIN_BASE_DIR}/lib
+
+for ssl_ver in libressl-2.5 libressl-3.0 openssl-0.9.8 openssl-1.0.0 openssl-1.0.1 openssl-1.0.1-fips openssl-1.0.2 openssl-1.1.0; do
+
+   cd ..
+   cp -a mongoc mongoc-${ssl_ver}
+   cd mongoc-${ssl_ver}
+   export PATH=$(readlink -f /opt/mongo-c-toolchain/${ssl_ver}/bin):${ADDL_PATH}:${OLD_PATH}
+   ssl_base_dir=$(readlink -f /opt/mongo-c-toolchain/${ssl_ver})
+   ssl_lib_dir=${ssl_base_dir}/lib
+   export EXTRA_CONFIGURE_FLAGS="-DCMAKE_VERBOSE_MAKEFILE=ON"
+   export EXTRA_CMAKE_PREFIX_PATH="${ssl_base_dir};${TOOLCHAIN_BASE_DIR}"
+
+   # Output some information about our build environment
+   "${CMAKE}" --version
+
+   # Run the build and tests
+   export LD_LIBRARY_PATH="${ssl_lib_dir}:${TOOLCHAIN_LIB_DIR}"
+   if [ "${ssl_ver#*libressl}" != "${ssl_ver}" ]; then
+      SSL_TYPE=LIBRESSL
+   else
+      SSL_TYPE=OPENSSL
+   fi
+   output_file=$(mktemp)
+   SSL=${SSL_TYPE} sh ./.evergreen/compile-unix.sh 2>&1 | tee -a "${output_file}"
+
+   # Verify that the toolchain components were used
+   if grep -Ec "[-]I/opt/mongo-c-toolchain/include" "${output_file}" >/dev/null \
+         && grep -Ec "[-]I/opt/mongo-c-toolchain/${ssl_ver}/include" "${output_file}" >/dev/null \
+         && grep -Ec "[-]L/opt/mongo-c-toolchain/lib" "${output_file}" >/dev/null \
+         && grep -Ec "/opt/mongo-c-toolchain/${ssl_ver}/lib" "${output_file}" >/dev/null; then
+      echo "Toolchain components for ${ssl_ver} were detected in build output...continuing."
+   else
+      echo "TOOLCHAIN COMPONENTS FOR ${ssl_ver} NOT DETECTED IN BUILD OUTPUT...ABORTING!"
+      exit 1
+   fi
+   rm -f "${output_file}"
+
+   cd ../mongoc
+done

.evergreen/compile-unix.sh

@@ -18,6 +18,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 # Options for CMake:
 #       LIBBSON                 Build against bundled or external libbson
 #       EXTRA_CONFIGURE_FLAGS   Extra configure flags to use
+#       EXTRA_CMAKE_PREFIX_PATH Extra directories to search for libraries/packages
 #       ZLIB                    Build against bundled or external zlib, or none
 #       SNAPPY                  Build against Snappy, or none
 #       SSL                     Build against OpenSSL or native or none
@@ -71,6 +72,11 @@ fi
 # as an environment variable (e.g. to force 32bit)
 [ -z "$MARCH" ] && MARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
 
+CMAKE_PREFIX_PATH="$INSTALL_DIR"
+if [ ! -z "$EXTRA_CMAKE_PREFIX_PATH" ]; then
+   CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;$EXTRA_CMAKE_PREFIX_PATH"
+fi
+
 # Default CMake flags for debug builds and release builds.
 # CMAKE_SKIP_RPATH avoids hardcoding absolute paths to dependency libraries.
 DEBUG_AND_RELEASE_FLAGS="\
@@ -82,7 +88,7 @@ DEBUG_AND_RELEASE_FLAGS="\
    -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF \
    -DENABLE_TRACING=$TRACING \
    -DENABLE_RDTSCP=$RDTSCP \
-   -DCMAKE_PREFIX_PATH=$INSTALL_DIR \
+   -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \
    -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
    -DENABLE_SHM_COUNTERS=$ENABLE_SHM_COUNTERS \
 "
@@ -212,7 +218,7 @@ if [ "$COMPILE_LIBMONGOCRYPT" = "ON" ]; then
    git clone https://github.com/mongodb/libmongocrypt
    mkdir libmongocrypt/cmake-build
    cd libmongocrypt/cmake-build
-   $CMAKE -DENABLE_SHARED_BSON=ON -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_PREFIX_PATH="$INSTALL_DIR" ../
+   $CMAKE -DENABLE_SHARED_BSON=ON -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" ../
    make install
    cd ../../
    else

.evergreen/config.yml

@@ -2016,6 +2016,24 @@ tasks:
         export SSL="OPENSSL"
         CC='${CC}' MARCH='${MARCH}' sh .evergreen/compile.sh
   - func: upload build
+- name: build-and-test-with-toolchain
+  commands:
+  - command: s3.get
+    params:
+      aws_key: '${toolchain_aws_key}'
+      aws_secret: '${toolchain_aws_secret}'
+      remote_file: 'mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz'
+      bucket: mongo-c-toolchain
+      local_file: mongo-c-toolchain.tar.gz
+  - command: shell.exec
+    type: test
+    params:
+      working_dir: mongoc
+      shell: bash
+      script: |-
+        set -o errexit
+        set -o xtrace
+        sh ./.evergreen/build-and-test-with-toolchain.sh
 - name: test-valgrind-latest-server-auth-nosasl-openssl
   tags:
   - latest
@@ -22482,6 +22500,9 @@ buildvariants:
     distros:
     - windows-64-vs2017-test
   - debug-compile-with-warnings
+  - name: build-and-test-with-toolchain
+    distros:
+    - debian10-small
 - name: clang34ubuntu
   display_name: clang 3.4 (Ubuntu 14.04)
   expansions:

build/evergreen_config_lib/tasks.py

@@ -436,7 +436,20 @@ def __init__(self, task_name, suffix_commands, orchestration=True, **kwargs):
                 CFLAGS='-fsanitize=thread -fno-omit-frame-pointer',
                 CHECK_LOG='ON',
                 SSL='OPENSSL',
-                EXTRA_CONFIGURE_FLAGS='-DENABLE_EXTRA_ALIGNMENT=OFF -DENABLE_SHM_COUNTERS=OFF')
+                EXTRA_CONFIGURE_FLAGS='-DENABLE_EXTRA_ALIGNMENT=OFF -DENABLE_SHM_COUNTERS=OFF'),
+    NamedTask('build-and-test-with-toolchain',
+              commands=[
+                  OD([('command', 's3.get'),
+                      ('params', OD([
+                          ('aws_key', '${toolchain_aws_key}'),
+                          ('aws_secret', '${toolchain_aws_secret}'),
+                          ('remote_file',
+                           'mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz'),
+                          ('bucket', 'mongo-c-toolchain'),
+                          ('local_file', 'mongo-c-toolchain.tar.gz'),
+                      ]))]),
+                  shell_mongoc('sh ./.evergreen/build-and-test-with-toolchain.sh')
+                  ])
 ]
 
 class IntegrationTask(MatrixTask):

build/evergreen_config_lib/variants.py

@@ -79,7 +79,9 @@ def days(n):
                  ('distros', ['windows-64-vs2017-test'])]),
              OD([('name', 'install-uninstall-check-msvc'),
                  ('distros', ['windows-64-vs2017-test'])]),
-             'debug-compile-with-warnings']),
+             'debug-compile-with-warnings',
+             OD([('name', 'build-and-test-with-toolchain'),
+                 ('distros', ['debian10-small'])]),]),
     Variant('clang34ubuntu',
             'clang 3.4 (Ubuntu 14.04)',
             'ubuntu1404-build',

src/libmongoc/CMakeLists.txt

@@ -122,6 +122,7 @@ if (NOT ENABLE_ZSTD STREQUAL OFF)
       set (MONGOC_ENABLE_COMPRESSION_ZSTD 1)
 
       include_directories (${ZSTD_INCLUDE_DIRS})
+      link_directories (${ZSTD_LIBRARY_DIRS})
       if (${CMAKE_VERSION} VERSION_LESS "3.12.0")
          set (MONGOC_ZSTD_LIBRARIES ${ZSTD_LIBRARIES})
       else ()