CDRIVER-4462 Use CMake 3.25.2 and improve compile scripts (#1205)

* CDRIVER-4462 Add find-cmake-version.h and find-cmake-latest.sh

* Use find-cmake-latest.sh in compile.sh scripts

* Print CMake version after search for verification

* Ensure build-and-test-with-toolchain uses the correct CMake version

* Ensure consistent CMake binary usage when compiling libmongocrypt

* Fix CMake linker selection issue on macos-1014

* Add required Evergreen expansions to tasks

* Extract scan-build compile script into compile-scan-build.sh

* Extract libmongocrypt compilation scripts into compile-libmongocrypt.sh

* Address several compile and scan-build warnings

* Reduce output verbosity of build-and-test-with-toolchain

* Relocate scan-build matrix to config_generator

* Use compile-libmongocrypt.sh in compile-std.sh

Author: eramongodb

.evergreen/config_generator/components/abi_compliance_check.py

@@ -11,6 +11,7 @@ class CheckABICompliance(Function):
     commands = [
         bash_exec(
             command_type=EvgCommandType.SETUP,
+            add_expansions_to_env=True,
             working_dir='mongoc',
             script='.evergreen/scripts/abi-compliance-check.sh'
         ),

.evergreen/config_generator/components/scan_build.py

@@ -0,0 +1,99 @@
+from shrub.v3.evg_build_variant import BuildVariant
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import FunctionCall
+from shrub.v3.evg_task import EvgTaskRef
+
+from config_generator.etc.distros import find_large_distro
+from config_generator.etc.distros import make_distro_str
+from config_generator.etc.distros import to_cc
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+from config_generator.etc.utils import Task
+
+
+TAG = 'scan-build-matrix'
+
+
+# pylint: disable=line-too-long
+# fmt: off
+MATRIX = [
+    ('macos-1014',       'clang', None  ),
+    ('ubuntu1404',       'clang', None  ),
+    ('ubuntu1604-arm64', 'clang', None  ),
+    ('ubuntu1604',       'clang', 'i686'),
+    ('ubuntu1604',       'clang', None  ),
+    ('ubuntu1804-arm64', 'clang', None  ),
+    ('ubuntu1804',       'clang', 'i686'),
+]
+# fmt: on
+# pylint: enable=line-too-long
+
+
+class ScanBuild(Function):
+    name = 'scan-build'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            add_expansions_to_env=True,
+            redirect_standard_error_to_output=True,
+            working_dir='mongoc',
+            script='.evergreen/scripts/compile-scan-build.sh',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return ScanBuild.defn()
+
+
+def tasks():
+    res = []
+
+    for distro_name, compiler, arch in MATRIX:
+        tags = [TAG, distro_name, compiler]
+
+        distro = find_large_distro(distro_name)
+
+        compile_vars = None
+        compile_vars = {'CC': to_cc(compiler)}
+
+        if arch:
+            tags.append(arch)
+            compile_vars.update({'MARCH': arch})
+
+        distro_str = make_distro_str(distro_name, compiler, arch)
+
+        task_name = f'scan-build-{distro_str}'
+
+        res.append(
+            Task(
+                name=task_name,
+                run_on=distro.name,
+                tags=tags,
+                commands=[
+                    ScanBuild.call(vars=compile_vars),
+                    FunctionCall(func='upload scan artifacts'),
+                ],
+            )
+        )
+
+    # TODO: remove once BUILD-16814 is resolved.
+    for task in res:
+         if task.run_on == 'macos-1014':
+            task.disable = True
+
+    return res
+
+
+def variants():
+    return [
+        BuildVariant(
+            name=TAG,
+            display_name=TAG,
+            tasks=[EvgTaskRef(name=f'.{TAG}')],
+        ),
+    ]

.evergreen/generated_configs/functions.yml

@@ -5,6 +5,7 @@ functions:
       params:
         binary: bash
         working_dir: mongoc
+        add_expansions_to_env: true
         args:
           - -c
           - .evergreen/scripts/abi-compliance-check.sh
@@ -406,6 +407,17 @@ functions:
         args:
           - -c
           - .evergreen/scripts/compile.sh
+  scan-build:
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: mongoc
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        args:
+          - -c
+          - .evergreen/scripts/compile-scan-build.sh
   std-compile:
     - command: subprocess.exec
       type: test

.evergreen/generated_configs/legacy-config.yml

@@ -472,34 +472,6 @@ tasks:
         set -o errexit
         env CFLAGS="-fno-omit-frame-pointer" CHECK_LOG="ON" DEBUG="ON" EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF" SANITIZE="address" SNAPPY="OFF" SSL="OPENSSL" ZLIB="BUNDLED" ZSTD="OFF" bash .evergreen/scripts/compile.sh
   - func: upload-build
-- name: debug-compile-scan-build
-  tags:
-  - clang
-  - debug-compile
-  - scan-build
-  - special
-  commands:
-  - command: shell.exec
-    type: test
-    params:
-      working_dir: mongoc
-      add_expansions_to_env: true
-      shell: bash
-      script: |-
-        set -o errexit
-        env ANALYZE="ON" CC="clang" DEBUG="ON" bash .evergreen/scripts/compile.sh
-  - func: upload-build
-  - func: upload scan artifacts
-  - command: shell.exec
-    type: test
-    params:
-      working_dir: mongoc
-      shell: bash
-      script: |-
-        set -o errexit
-        if find scan -name \*.html | grep -q html; then
-          exit 123
-        fi
 - name: compile-tracing
   commands:
   - command: shell.exec
@@ -9025,6 +8997,7 @@ tasks:
   - func: fetch-source
   - command: shell.exec
     params:
+      add_expansions_to_env: true
       shell: bash
       script: |-
         set -o errexit
@@ -9066,6 +9039,7 @@ tasks:
   - func: fetch-source
   - command: shell.exec
     params:
+      add_expansions_to_env: true
       shell: bash
       script: |-
         set -o errexit
@@ -9089,6 +9063,7 @@ tasks:
   - func: fetch-source
   - command: shell.exec
     params:
+      add_expansions_to_env: true
       shell: bash
       script: |-
         set -o errexit
@@ -9127,6 +9102,7 @@ tasks:
   commands:
   - command: shell.exec
     params:
+      add_expansions_to_env: true
       shell: bash
       script: |-
         set -o errexit
@@ -9287,7 +9263,6 @@ buildvariants:
     CC: clang
   run_on: ubuntu1404-build
   tasks:
-  - debug-compile-scan-build
   - release-compile
   - debug-compile-rdtscp
 - name: clang35
@@ -9337,7 +9312,6 @@ buildvariants:
     MARCH: i686
   run_on: ubuntu1804-test
   tasks:
-  - debug-compile-scan-build
   - release-compile
   - debug-compile-nosasl-nossl
   - debug-compile-no-align
@@ -9350,7 +9324,6 @@ buildvariants:
     MARCH: i686
   run_on: ubuntu1604-test
   tasks:
-  - debug-compile-scan-build
   - release-compile
   - debug-compile-no-align
 - name: clang38ubuntu
@@ -9360,7 +9333,6 @@ buildvariants:
   run_on: ubuntu1604-test
   tasks:
   - .compression !.zstd
-  - debug-compile-scan-build
   - release-compile
   - debug-compile-sasl-openssl
   - debug-compile-nosasl-openssl
@@ -9503,7 +9475,6 @@ buildvariants:
   - debug-compile-nosrv
   - debug-compile-sasl-darwinssl
   - debug-compile-nosasl-nossl
-  - .debug-compile .clang
   - .authentication-tests .darwinssl
   - .latest .nossl
   - test-dns-darwinssl
@@ -9616,7 +9587,6 @@ buildvariants:
   run_on: ubuntu1804-arm64-large
   tasks:
   - .compression !.snappy !.zstd
-  - debug-compile-scan-build
   - debug-compile-no-align
   - release-compile
   - debug-compile-nosasl-nossl
@@ -9633,7 +9603,6 @@ buildvariants:
   run_on: ubuntu1604-arm64-large
   tasks:
   - .compression !.snappy !.zstd
-  - debug-compile-scan-build
   - debug-compile-no-align
   - release-compile
   batchtime: 1440

.evergreen/generated_configs/tasks.yml

@@ -3194,6 +3194,65 @@ tasks:
       - func: fetch-det
       - func: bootstrap-mongo-orchestration
       - func: run-tests
+  - name: scan-build-macos-1014-clang
+    run_on: macos-1014
+    tags: [scan-build-matrix, macos-1014, clang]
+    disable: true
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1404-clang
+    run_on: ubuntu1404-large
+    tags: [scan-build-matrix, ubuntu1404, clang]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1604-arm64-clang
+    run_on: ubuntu1604-arm64-large
+    tags: [scan-build-matrix, ubuntu1604-arm64, clang]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1604-clang
+    run_on: ubuntu1604-large
+    tags: [scan-build-matrix, ubuntu1604, clang]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1604-clang-i686
+    run_on: ubuntu1604-large
+    tags: [scan-build-matrix, ubuntu1604, clang, i686]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+          MARCH: i686
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1804-arm64-clang
+    run_on: ubuntu1804-arm64-large
+    tags: [scan-build-matrix, ubuntu1804-arm64, clang]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+      - func: upload scan artifacts
+  - name: scan-build-ubuntu1804-clang-i686
+    run_on: ubuntu1804-large
+    tags: [scan-build-matrix, ubuntu1804, clang, i686]
+    commands:
+      - func: scan-build
+        vars:
+          CC: clang
+          MARCH: i686
+      - func: upload scan artifacts
   - name: std-c11-archlinux-clang-compile
     run_on: archlinux-large
     tags: [std-matrix, archlinux, clang, compile, std-c11]

.evergreen/generated_configs/variants.yml

@@ -67,6 +67,10 @@ buildvariants:
       DEBUG: "ON"
     tasks:
       - name: .sasl-matrix-winssl
+  - name: scan-build-matrix
+    display_name: scan-build-matrix
+    tasks:
+      - name: .scan-build-matrix
   - name: std-matrix
     display_name: std-matrix
     tasks:

.evergreen/legacy_config_generator/evergreen_config_generator/functions.py

@@ -56,6 +56,7 @@ def shell_exec(script,
                working_dir=None,
                background=False,
                add_expansions_to_env=False,
+               redirect_standard_error_to_output=False,
                ):
     dedented = ''
     if errexit:
@@ -85,6 +86,9 @@ def shell_exec(script,
     if add_expansions_to_env:
         command['params']['add_expansions_to_env'] = True
 
+    if redirect_standard_error_to_output:
+        command['params']['redirect_standard_error_to_output'] = True
+
     command['params']['shell'] = 'bash'
     command['params']['script'] = dedented
     return command

.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py

@@ -184,17 +184,6 @@ def __init__(self, task_name, suffix_commands, orchestration=True, **kwargs):
                 sanitize=['address'],
                 EXTRA_CONFIGURE_FLAGS="-DENABLE_EXTRA_ALIGNMENT=OFF",
                 SSL='OPENSSL'),
-    SpecialTask('debug-compile-scan-build',
-                tags=['clang', 'debug-compile', 'scan-build'],
-                continue_on_err=True,
-                ANALYZE='ON',
-                CC='clang',
-                suffix_commands=[
-                    func('upload scan artifacts'),
-                    shell_mongoc('''
-                        if find scan -name \*.html | grep -q html; then
-                          exit 123
-                        fi''')]),
     CompileTask('compile-tracing',
                 TRACING='ON', CFLAGS='-Werror -Wno-cast-align'),
     CompileTask('release-compile',

.evergreen/legacy_config_generator/evergreen_config_lib/testazurekms.py

@@ -51,7 +51,7 @@ def _create_tasks():
           AZUREKMS_CMD="tar xf testazurekms.tgz" \
             $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
           echo "Untarring file ... end"
-        ''', test=False),
+        ''', test=False, add_expansions_to_env=True),
         shell_exec (r'''
           export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
           export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
@@ -69,7 +69,7 @@ def _create_tasks():
           pushd mongoc
           ./.evergreen/scripts/compile-test-azurekms.sh
           popd
-        ''', test=False),
+        ''', test=False, add_expansions_to_env=True),
         shell_exec (r'''
           LD_LIBRARY_PATH=$PWD/install \
           MONGODB_URI='mongodb://localhost:27017' \