CXX-2162 Use DET hygienic scripts for Python binary selection and venv creation (#913)

* Replace activate_venv.sh with activate-kmstlsvenv.sh

* Replace manual Python selection in compile function

* Use venv-utils.sh in stop_mongod function

* Format collection.cpp and uri.cpp to address lint task failure

* Guard cd to potentially missing DET repo

* CXX-2628 Apply Python >=3.10 selection workaround used by C Driver

* Remove special-casing of TLS test cert files

Author: eramongodb

.mci.yml

@@ -207,13 +207,12 @@ functions:
             script: |
                 set -o errexit
                 set -o pipefail
-                cd drivers-evergreen-tools/.evergreen/orchestration
-                if [ -f venv/bin/activate ]; then
-                    . venv/bin/activate
-                elif [ -f venv/Scripts/activate ]; then
-                    . venv/Scripts/activate
+                if cd drivers-evergreen-tools/.evergreen/orchestration; then
+                  . ../venv-utils.sh
+                  if venvactivate venv; then
+                    mongo-orchestration stop
+                  fi
                 fi
-                mongo-orchestration stop
 
     "install_c_driver":
       - command: expansions.update
@@ -280,6 +279,30 @@ functions:
             fi
 
     "run_kms_servers":
+      - command: shell.exec
+        params:
+          shell: bash
+          script: |-
+            set -o errexit
+            echo "Preparing CSFLE venv environment..."
+            cd ./drivers-evergreen-tools/.evergreen/csfle
+            # This function ensures future invocations of activate-kmstlsvenv.sh conducted in
+            # parallel do not race to setup a venv environment; it has already been prepared.
+            # This primarily addresses the situation where the "test" and "run_kms_servers"
+            # functions invoke 'activate-kmstlsvenv.sh' simultaneously.
+            if [[ "$OSTYPE" =~ cygwin && ! -d kmstlsvenv ]]; then
+                # Avoid using Python 3.10 on Windows due to incompatible cipher suites.
+                # See CXX-2628.
+                . ../venv-utils.sh
+                venvcreate "C:/python/Python39/python.exe" kmstlsvenv || # windows-2017
+                venvcreate "C:/python/Python38/python.exe" kmstlsvenv    # windows-2015
+                python -m pip install --upgrade boto3~=1.19 pykmip~=0.10.0
+                deactivate
+            else
+                . ./activate-kmstlsvenv.sh
+                deactivate
+            fi
+            echo "Preparing CSFLE venv environment... done."
       - command: shell.exec
         params:
           background: true
@@ -288,7 +311,7 @@ functions:
             set -o errexit
             echo "Starting mock KMS servers..."
             cd ./drivers-evergreen-tools/.evergreen/csfle
-            . ./activate_venv.sh
+            . ./activate-kmstlsvenv.sh
             python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8999 &
             python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 &
             python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 &
@@ -326,21 +349,14 @@ functions:
                   fi
                   export CMAKE=${cmake}
 
-                  if [ "x$(lsb_release -cs)" = "xtrusty" -a -f /opt/mongodbtoolchain/v2/bin/python ]; then
-                      /opt/mongodbtoolchain/v2/bin/python -m virtualenv venv
-                  elif ! python -m virtualenv venv 2>/dev/null; then
-                      /opt/mongodbtoolchain/v3/bin/python3 -m venv venv
+                  if [ ! -d ../drivers-evergreen-tools ]; then
+                    git clone --depth 1 [email protected]:mongodb-labs/drivers-evergreen-tools.git ../drivers-evergreen-tools
                   fi
+                  . ../drivers-evergreen-tools/.evergreen/find-python3.sh
+                  . ../drivers-evergreen-tools/.evergreen/venv-utils.sh
 
-                  cd venv
-                  if [ -f bin/activate ]; then
-                      . bin/activate
-                      ./bin/pip install GitPython
-                  elif [ -f Scripts/activate ]; then
-                      . Scripts/activate
-                      ./Scripts/pip install GitPython
-                  fi
-                  cd ..
+                  venvcreate "$(find_python3)" venv
+                  python -m pip install GitPython
 
                   export GENERATOR="${generator}"
 
@@ -417,14 +433,35 @@ functions:
                       # export environment variables for encryption tests
                       set +o errexit
 
-                      if [ "Windows_NT" == "$OS"]; then
-                          export MONGOCXX_TEST_CSFLE_TLS_CA_FILE=$DRIVERS_TOOLS\.evergreen\x509gen\ca.pem
-                          export MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=$DRIVERS_TOOLS\.evergreen\x509gen\client.pem
-                      else
-                          export MONGOCXX_TEST_CSFLE_TLS_CA_FILE=$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem
-                          export MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=$DRIVERS_TOOLS/.evergreen/x509gen/client.pem
+                      # Avoid printing credentials in logs.
+                      set +o xtrace
+
+                      echo "Setting temporary credentials..."
+                      pushd "$DRIVERS_TOOLS/.evergreen/csfle"
+                      export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
+                      export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
+                      export AWS_DEFAULT_REGION="us-east-1"
+                      echo "Running activate-kmstlsvenv.sh..."
+                      . ./activate-kmstlsvenv.sh
+                      echo "Running activate-kmstlsvenv.sh... done."
+                      echo "Running set-temp-creds.sh..."
+                      . ./set-temp-creds.sh
+                      echo "Running set-temp-creds.sh... done."
+                      deactivate
+                      popd # "$DRIVERS_TOOLS/.evergreen/csfle"
+                      echo "Setting temporary credentials... done."
+
+                      # Ensure temporary credentials were properly set.
+                      if [ -z "$CSFLE_AWS_TEMP_ACCESS_KEY_ID" ]; then
+                        echo "Failed to set temporary credentials!"
+                        exit 1
                       fi
 
+                      export MONGOCXX_TEST_CSFLE_TLS_CA_FILE=$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem
+                      export MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=$DRIVERS_TOOLS/.evergreen/x509gen/client.pem
+                      export MONGOCXX_TEST_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID"
+                      export MONGOCXX_TEST_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY"
+                      export MONGOCXX_TEST_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN"
                       export MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
                       export MONGOCXX_TEST_AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
                       export MONGOCXX_TEST_AZURE_TENANT_ID="${cse_azure_tenant_id}"