CDRIVER-4134 Use simple HTTP server for /http tests (#1237)

* Add simple_http_server.py script for /http tests

* Add missing package requirement "packaging"

* Use simple HTTP server for /http tests

* Reuse result of find_python3 across config

Author: eramongodb

.evergreen/config_generator/components/funcs/fetch_det.py

@@ -1,4 +1,5 @@
 from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import expansions_update
 
 from config_generator.etc.function import Function
 from config_generator.etc.utils import bash_exec
@@ -15,6 +16,21 @@ class FetchDET(Function):
                 fi
             ''',
         ),
+
+        # python is used frequently enough by many tasks that it is worth
+        # running find_python3 once here and reusing the result.
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            script='''\
+                set -o errexit
+                . drivers-evergreen-tools/.evergreen/find-python3.sh
+                echo "PYTHON3_BINARY: $(find_python3)" >|python3_binary.yml
+            ''',
+        ),
+        expansions_update(
+            command_type=EvgCommandType.SETUP,
+            file='python3_binary.yml',
+        ),
     ]
 
     @classmethod

.evergreen/config_generator/components/funcs/run_simple_http_server.py

@@ -0,0 +1,31 @@
+from shrub.v3.evg_command import EvgCommandType
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class RunSimpleHTTPServer(Function):
+    name = 'run-simple-http-server'
+    command_type = EvgCommandType.SETUP
+    commands = [
+        bash_exec(
+            command_type=command_type,
+            background=True,
+            working_dir='mongoc',
+            script='''\
+                set -o errexit
+                echo "Starting simple HTTP server..."
+                command -V "${PYTHON3_BINARY}" >/dev/null
+                "${PYTHON3_BINARY}" .evergreen/scripts/simple_http_server.py
+                echo "Starting simple HTTP server... done."
+            ''',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return RunSimpleHTTPServer.defn()

.evergreen/config_generator/etc/sanitizers/test.py

@@ -12,6 +12,7 @@
 from config_generator.components.funcs.bootstrap_mongo_orchestration import BootstrapMongoOrchestration
 from config_generator.components.funcs.fetch_build import FetchBuild
 from config_generator.components.funcs.fetch_det import FetchDET
+from config_generator.components.funcs.run_simple_http_server import RunSimpleHTTPServer
 from config_generator.components.funcs.run_mock_kms_servers import RunMockKMSServers
 from config_generator.components.funcs.run_tests import RunTests
 
@@ -30,7 +31,10 @@ def generate_test_tasks(SSL, TAG, MATRIX, MORE_COMPILE_TAGS=None, MORE_TEST_TAGS
     MORE_VARS = MORE_VARS if MORE_VARS else {}
 
     for distro_name, compiler, arch, sasl, auths, topologies, server_vers in MATRIX:
-        tags = [TAG, 'test', distro_name, compiler, f'sasl-{sasl}'] + MORE_COMPILE_TAGS
+        tags = [
+            TAG, 'test', distro_name, compiler, f'sasl-{sasl}'
+        ] + MORE_COMPILE_TAGS
+
         test_distro = find_small_distro(distro_name)
 
         compile_vars = []
@@ -77,6 +81,7 @@ def generate_test_tasks(SSL, TAG, MATRIX, MORE_COMPILE_TAGS=None, MORE_TEST_TAGS
             test_commands.append(expansions_update(updates=updates))
             test_commands.append(FetchDET.call())
             test_commands.append(BootstrapMongoOrchestration.call())
+            test_commands.append(RunSimpleHTTPServer.call())
 
             if 'cse' in MORE_COMPILE_TAGS:
                 test_commands.append(RunMockKMSServers.call())

.evergreen/config_generator/etc/sasl/test.py

@@ -12,6 +12,7 @@
 from config_generator.components.funcs.bootstrap_mongo_orchestration import BootstrapMongoOrchestration
 from config_generator.components.funcs.fetch_build import FetchBuild
 from config_generator.components.funcs.fetch_det import FetchDET
+from config_generator.components.funcs.run_simple_http_server import RunSimpleHTTPServer
 from config_generator.components.funcs.run_tests import RunTests
 
 
@@ -56,6 +57,7 @@ def generate_test_tasks(SSL, TAG, MATRIX):
             test_commands.append(expansions_update(updates=updates))
             test_commands.append(FetchDET.call())
             test_commands.append(BootstrapMongoOrchestration.call())
+            test_commands.append(RunSimpleHTTPServer.call())
             test_commands.append(RunTests.call())
 
             res.append(

.evergreen/config_generator/requirements.txt

@@ -1,5 +1,6 @@
 -e git+https://github.com/mongodb-labs/drivers-evergreen-tools#egg=evergreen_config_generator&subdirectory=evergreen_config_generator
 git-url-parse==1.2.2
+packaging==23.1
 pbr==5.11.1
 pydantic==1.10.4
 PyYAML==5.4.1

.evergreen/generated_configs/functions.yml

@@ -173,6 +173,20 @@ functions:
             if [[ ! -d drivers-evergreen-tools ]]; then
                 git clone --depth=1 [email protected]:mongodb-labs/drivers-evergreen-tools.git
             fi
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        args:
+          - -c
+          - |
+            set -o errexit
+            . drivers-evergreen-tools/.evergreen/find-python3.sh
+            echo "PYTHON3_BINARY: $(find_python3)" >|python3_binary.yml
+    - command: expansions.update
+      type: setup
+      params:
+        file: python3_binary.yml
   fetch-source:
     - command: git.get_project
       type: setup
@@ -318,6 +332,21 @@ functions:
             python -u kms_kmip_server.py &
             deactivate
             echo "Starting mock KMS TLS servers... done."
+  run-simple-http-server:
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        working_dir: mongoc
+        background: true
+        args:
+          - -c
+          - |
+            set -o errexit
+            echo "Starting simple HTTP server..."
+            command -V "${PYTHON3_BINARY}" >/dev/null
+            "${PYTHON3_BINARY}" .evergreen/scripts/simple_http_server.py
+            echo "Starting simple HTTP server... done."
   run-tests:
     - command: subprocess.exec
       type: test

.evergreen/generated_configs/legacy-config.yml

@@ -1165,6 +1165,7 @@ tasks:
       MONGODB_VERSION: latest
       SSL: openssl
       TOPOLOGY: replica_set
+  - func: run-simple-http-server
   - func: fetch-det
   - func: run-mock-kms-servers
   - func: run-tests
@@ -1326,6 +1327,7 @@ tasks:
       AUTH: noauth
       ORCHESTRATION_FILE: zlib
       SSL: nossl
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: noauth
@@ -1348,6 +1350,7 @@ tasks:
       AUTH: noauth
       ORCHESTRATION_FILE: snappy
       SSL: nossl
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: noauth
@@ -1370,6 +1373,7 @@ tasks:
       AUTH: noauth
       ORCHESTRATION_FILE: zstd
       SSL: nossl
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: noauth
@@ -1394,6 +1398,7 @@ tasks:
       AUTH: noauth
       ORCHESTRATION_FILE: snappy-zlib-zstd
       SSL: nossl
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: noauth
@@ -1411,6 +1416,7 @@ tasks:
     vars:
       MONGODB_VERSION: latest
       TOPOLOGY: server
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       URI: mongodb://localhost/?retryWrites=true
@@ -1429,6 +1435,7 @@ tasks:
     vars:
       MONGODB_VERSION: latest
       TOPOLOGY: server
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       URI: null
@@ -1556,6 +1563,7 @@ tasks:
       REQUIRE_API_VERSION: 'true'
       SSL: ssl
       TOPOLOGY: server
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: auth
@@ -1578,6 +1586,7 @@ tasks:
       ORCHESTRATION_FILE: versioned-api-testing
       SSL: nossl
       TOPOLOGY: server
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       AUTH: noauth
@@ -1718,6 +1727,7 @@ tasks:
   - func: bootstrap-mongo-orchestration
     vars:
       IPV4_ONLY: 'off'
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       IPV4_ONLY: 'off'
@@ -1739,6 +1749,7 @@ tasks:
   - func: bootstrap-mongo-orchestration
     vars:
       IPV4_ONLY: 'off'
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       IPV4_ONLY: 'off'
@@ -1760,6 +1771,7 @@ tasks:
   - func: bootstrap-mongo-orchestration
     vars:
       IPV4_ONLY: 'on'
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       IPV4_ONLY: 'on'
@@ -1781,6 +1793,7 @@ tasks:
   - func: bootstrap-mongo-orchestration
     vars:
       IPV4_ONLY: 'on'
+  - func: run-simple-http-server
   - func: run-tests
     vars:
       IPV4_ONLY: 'on'
@@ -8958,6 +8971,7 @@ tasks:
       MONGODB_VERSION: '5.0'
       SSL: ssl
       TOPOLOGY: sharded_cluster
+  - func: run-simple-http-server
   - func: start load balancer
     vars:
       MONGODB_URI: mongodb://localhost:27017,localhost:27018
@@ -8985,6 +8999,7 @@ tasks:
       MONGODB_VERSION: latest
       SSL: ssl
       TOPOLOGY: sharded_cluster
+  - func: run-simple-http-server
   - func: start load balancer
     vars:
       MONGODB_URI: mongodb://localhost:27017,localhost:27018
@@ -9012,6 +9027,7 @@ tasks:
       MONGODB_VERSION: '5.0'
       SSL: nossl
       TOPOLOGY: sharded_cluster
+  - func: run-simple-http-server
   - func: start load balancer
     vars:
       MONGODB_URI: mongodb://localhost:27017,localhost:27018
@@ -9039,6 +9055,7 @@ tasks:
       MONGODB_VERSION: latest
       SSL: nossl
       TOPOLOGY: sharded_cluster
+  - func: run-simple-http-server
   - func: start load balancer
     vars:
       MONGODB_URI: mongodb://localhost:27017,localhost:27018