CXX-2710 Run microbenchmarks in Evergreen (#1063)

---------

Co-authored-by: Ezra Chung <[email protected]>
Co-authored-by: Kyle Kloberdanz <[email protected]>

Author: 3 people

.mci.yml

@@ -360,6 +360,16 @@ functions:
                   CXX: ${cxx_compiler}
               script: .evergreen/compile.sh
 
+    "compile_benchmarks":
+      - command: shell.exec
+        type: setup
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: |
+            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(pwd)/../mongoc" -DCMAKE_CXX_STANDARD=20
+            cmake --build build --target microbenchmarks --parallel 64
+
     "test":
         - command: shell.exec
           params:
@@ -509,6 +519,24 @@ functions:
             make -C extras/docker/bookworm nocachebuild test
             echo "Building Red Hat UBI Docker image"
             make -C extras/docker/redhat-ubi-9.3 nocachebuild test
+    
+    "run benchmarks":
+      - command: shell.exec
+        type: setup
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: ./etc/microbenchmark-test-data.sh
+      - command: shell.exec
+        type: test
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: ./build/benchmark/microbenchmarks all
+      - command: perf.send
+        params:
+          name: perf
+          file: mongo-cxx-driver/results.json
 
 #######################################
 #              Post Task              #
@@ -620,6 +648,14 @@ tasks:
         vars:
             ENABLE_TESTS: OFF
 
+    - name: compile_and_run_benchmarks
+      commands:
+      - func: "setup"
+      - func: "start_mongod"
+      - func: "fetch_c_driver_source"
+      - func: "compile_benchmarks"
+      - func: "run benchmarks"
+
     - name: compile_macro_guard_tests
       commands:
       - func: "setup"
@@ -1303,6 +1339,14 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
 
+    - name: benchmarks-rhel9
+      display_name: "Benchmarks (RHEL 9.2)"
+      expansions:
+          mongodb_version: "v6.0-perf"
+      run_on: rhel90-dbx-perf-large
+      tasks:
+          - name: compile_and_run_benchmarks
+
     - name: arm-rhel9-release-latest
       display_name: "arm64 RHEL 9 Release (MongoDB Latest)"
       expansions:

benchmark/CMakeLists.txt

@@ -50,7 +50,7 @@ file (GLOB benchmark_DIST_hpps RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.hpp)
 
 set_dist_list (benchmark_DIST
    CMakeLists.txt
-   README.txt
+   README.md
    ${BENCHMARK_LIBRARY}
    ${benchmark_DIST_hpps}
 )

benchmark/README.txt renamed to benchmark/README.md

@@ -1,10 +1,16 @@
 This suite implements the benchmarks described in this spec: https://docs.google.com/document/d/1x7dCHHx_owOFYIPZ6VQwPerSWRhdkMSiepyFrNSFlLw/edit#
 
-In order to run the microbenchmarks, first run etc/microbenchmark-test-data.sh to download the data.
+# Compiling
+Configure the C++ driver to build with C++17 or newer and build the `microbenchmarks` target.
 
-In order to run specific tests, just specify their names as arguments. If run with no arguments,
-all benchmarks will be run.
-e.g. build/benchmark/microbenchmarks BSONBench MultiBench
+# Running
+In order to run the microbenchmarks, first run `etc/microbenchmark-test-data.sh` to download the test data.
+
+In order to run specific tests, just specify their names as arguments:
+`build/benchmark/microbenchmarks BSONBench MultiBench`
+
+To run all tests, specify `all` as an argument:
+`build/benchmark/microbenchmarks all`
 
 Full list of options:
 BSONBench
@@ -14,14 +20,9 @@ ReadBench
 WriteBench
 RunCommandBench
 
-Note: make sure you run both the download script and the microbenchmarks binary from the project root.
-
-See the spec for details on these benchmarks.
-
-In case there is trouble running the benchmarks, they were written and will run with: 
- -libbson at commit 1c5b90022bbbdf0fa8095b337a25b218a2651241 
- -mongoc driver at commit a3c8a760ce90f144fd65d8a4a6e3606cbeea2f6b
+Note: run both the download script and the microbenchmarks binary from the project root.
 
+# Notes
 Note that in order to compare against the other drivers, an inMemory mongod instance should be 
 used.
 

benchmark/benchmark_runner.cpp

@@ -14,6 +14,14 @@
 
 #include "benchmark_runner.hpp"
 
+#include <chrono>
+#include <cstdint>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+
 #include "bson/bson_encoding.hpp"
 #include "multi_doc/bulk_insert.hpp"
 #include "multi_doc/find_many.hpp"
@@ -26,7 +34,13 @@
 #include "single_doc/find_one_by_id.hpp"
 #include "single_doc/insert_one.hpp"
 #include "single_doc/run_command.hpp"
+#include <bsoncxx/builder/basic/array.hpp>
+#include <bsoncxx/builder/basic/document.hpp>
+#include <bsoncxx/builder/basic/kvp.hpp>
+#include <bsoncxx/builder/basic/sub_document.hpp>
+#include <bsoncxx/json.hpp>
 #include <bsoncxx/stdx/make_unique.hpp>
+#include <bsoncxx/types.hpp>
 
 namespace benchmark {
 
@@ -47,26 +61,28 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
     _microbenches.push_back(make_unique<run_command>());
     _microbenches.push_back(make_unique<find_one_by_id>("single_and_multi_document/tweet.json"));
     _microbenches.push_back(make_unique<insert_one>(
-        "TestSmallDocInsertOne", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+        "TestSmallDocInsertOne", 2.75, iterations, "single_and_multi_document/small_doc.json"));
     _microbenches.push_back(make_unique<insert_one>(
         "TestLargeDocInsertOne", 27.31, 10, "single_and_multi_document/large_doc.json"));
 
     // Multi doc microbenchmarks
     _microbenches.push_back(make_unique<find_many>("single_and_multi_document/tweet.json"));
     _microbenches.push_back(make_unique<bulk_insert>(
-        "TestSmallDocBulkInsert", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+        "TestSmallDocBulkInsert", 2.75, iterations, "single_and_multi_document/small_doc.json"));
     _microbenches.push_back(make_unique<bulk_insert>(
         "TestLargeDocBulkInsert", 27.31, 10, "single_and_multi_document/large_doc.json"));
-    _microbenches.push_back(
-        make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
-    _microbenches.push_back(
-        make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
+    // CXX-2794: Disable GridFS benchmarks due to long runtime
+    // _microbenches.push_back(
+    //     make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
+    // _microbenches.push_back(
+    //     make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
 
     // Parallel microbenchmarks
     _microbenches.push_back(make_unique<json_multi_import>("parallel/ldjson_multi"));
     _microbenches.push_back(make_unique<json_multi_export>("parallel/ldjson_multi"));
-    _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
-    _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
+    // CXX-2794: Disable GridFS benchmarks due to long runtime
+    // _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
+    // _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
 
     // Need to remove some
     if (!_types.empty()) {
@@ -89,8 +105,7 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
 }
 
 void benchmark_runner::run_microbenches() {
-    mongocxx::instance instance{};
-
+    _start_time = std::chrono::system_clock::now();
     for (std::unique_ptr<microbench>& bench : _microbenches) {
         std::cout << "Starting " << bench->get_name() << "..." << std::endl;
 
@@ -103,6 +118,7 @@ void benchmark_runner::run_microbenches() {
                   << " second(s) | " << score.get_score() << " MB/s" << std::endl
                   << std::endl;
     }
+    _end_time = std::chrono::system_clock::now();
 }
 
 double benchmark_runner::calculate_average(benchmark_type tag) {
@@ -145,20 +161,48 @@ double benchmark_runner::calculate_driver_bench_score() {
     return (calculate_read_bench_score() + calculate_write_bench_score()) / 2.0;
 }
 
-void benchmark_runner::print_scores() {
+void benchmark_runner::write_scores() {
     double read = -1;
     double write = -1;
 
+    using namespace bsoncxx;
+    using builder::basic::sub_document;
+
+    auto doc = builder::basic::document{};
+    doc.append(kvp("info", [](sub_document subdoc) {
+        subdoc.append(kvp("test_name", "C++ Driver microbenchmarks"));
+    }));
+
+    auto write_time =
+        [](const std::chrono::time_point<std::chrono::system_clock> t) -> std::string {
+        std::time_t t1 = std::chrono::system_clock::to_time_t(t);
+        std::ostringstream oss;
+        oss << std::put_time(std::gmtime(&t1), "%Y-%m-%dT%H:%M:%S") << "+00:00";
+        return oss.str();
+    };
+    doc.append(kvp("created_at", write_time(_start_time)));
+    doc.append(kvp("completed_at", write_time(_end_time)));
+    doc.append(kvp("artifacts", builder::basic::make_array()));
+
+    auto metrics_array = builder::basic::array{};
+    std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+
     std::cout << "Individual microbenchmark scores:" << std::endl << "===========" << std::endl;
     for (auto&& bench : _microbenches) {
         auto& score = bench->get_results();
+        const auto bench_time = static_cast<double>(score.get_percentile(50).count()) / 1000.0;
 
-        std::cout << bench->get_name() << ": "
-                  << static_cast<double>(score.get_percentile(50).count()) / 1000.0
-                  << " second(s) | " << score.get_score() << " MB/s" << std::endl;
-    }
+        std::cout << bench->get_name() << ": " << bench_time << " seconds | " << score.get_score()
+                  << " MB/s" << std::endl;
 
-    std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+        auto metric_doc = builder::basic::document{};
+        metric_doc.append(kvp("name", bench->get_name()));
+        metric_doc.append(kvp("type", "THROUGHPUT"));
+        metric_doc.append(kvp("value", score.get_score()));
+        metrics_array.append(metric_doc);
+    }
+    doc.append(kvp("metrics", metrics_array));
+    doc.append(kvp("sub_tests", builder::basic::make_array()));
 
     auto print_comp = [this, &read, &write](benchmark_type type) {
         double avg = calculate_average(type);
@@ -169,7 +213,7 @@ void benchmark_runner::print_scores() {
             write = avg;
         }
 
-        std::cout << type_names[type] << " " << avg << " MB/s" << std::endl;
+        std::cout << type_names.at(type) << " " << avg << " MB/s" << std::endl;
     };
 
     if (!_types.empty()) {
@@ -185,5 +229,8 @@ void benchmark_runner::print_scores() {
     if (read > 0 && write > 0) {
         std::cout << "DriverBench: " << (read + write) / 2.0 << " MB/s" << std::endl;
     }
+
+    std::ofstream os{"results.json"};
+    os << '[' << bsoncxx::to_json(doc.view()) << ']';
 }
 }  // namespace benchmark

benchmark/benchmark_runner.hpp

@@ -14,10 +14,11 @@
 
 #pragma once
 
-#include <bsoncxx/stdx/optional.hpp>
-#include <mongocxx/instance.hpp>
+#include <chrono>
 
 #include "microbench.hpp"
+#include <bsoncxx/stdx/optional.hpp>
+#include <mongocxx/instance.hpp>
 
 namespace benchmark {
 
@@ -27,7 +28,7 @@ class benchmark_runner {
 
     void run_microbenches();
 
-    void print_scores();
+    void write_scores();
 
     double calculate_bson_bench_score();
 
@@ -46,6 +47,8 @@ class benchmark_runner {
    private:
     double calculate_average(benchmark_type);
 
+    std::chrono::time_point<std::chrono::system_clock> _start_time;
+    std::chrono::time_point<std::chrono::system_clock> _end_time;
     std::vector<std::unique_ptr<microbench>> _microbenches;
     std::set<benchmark_type> _types;
 };

benchmark/bson/bson_decoding.hpp

@@ -42,7 +42,7 @@ void bson_decoding::setup() {
 }
 
 void bson_decoding::task() {
-    for (std::uint32_t i = 0; i < 10000; i++) {
+    for (std::uint32_t i = 0; i < iterations; i++) {
         // TODO CXX-1241: call bson_as_extended json on _json.
     }
 }

benchmark/bson/bson_encoding.hpp

@@ -16,11 +16,10 @@
 
 #include <iostream>
 
+#include "../microbench.hpp"
 #include <bsoncxx/json.hpp>
 #include <bsoncxx/types.hpp>
 
-#include "../microbench.hpp"
-
 namespace benchmark {
 
 class bson_encoding : public microbench {
@@ -58,7 +57,7 @@ void visit_document(bsoncxx::document::view doc) {
 
 // Mirroring mongo-c-driver's interpretation of the spec.
 void bson_encoding::task() {
-    for (std::uint32_t i = 0; i < 10000; i++) {
+    for (std::uint32_t i = 0; i < iterations; i++) {
         visit_document(_doc->view());
     }
 }

benchmark/main.cpp

@@ -12,34 +12,46 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+static_assert(__cplusplus >= 201703L, "requires C++17 or higher");
+
+#include <chrono>
 #include <iostream>
+#include <stdexcept>
 
 #include "benchmark_runner.hpp"
+#include <bsoncxx/stdx/string_view.hpp>
+#include <mongocxx/instance.hpp>
 
 using namespace benchmark;
 
 int main(int argc, char* argv[]) {
+    mongocxx::instance instance;
     std::set<benchmark_type> types;
 
     if (argc > 1) {
-        for (int x = 1; x < argc; ++x) {
-            std::string type{argv[x]};
-            auto it = names_types.find(type);
-
-            if (it != names_types.end()) {
+        if (bsoncxx::stdx::string_view(argv[1]) == "all") {
+            for (const auto& [name, type] : names_types) {
+                types.insert(type);
+            }
+        } else {
+            for (int x = 1; x < argc; ++x) {
+                std::string type{argv[x]};
+                auto it = names_types.find(type);
+
+                if (it == names_types.end()) {
+                    throw std::runtime_error("Invalid benchmark: " + type);
+                }
                 types.insert(it->second);
-            } else {
-                std::cerr << "Invalid benchmark: " << type << std::endl;
             }
         }
 
         if (types.empty()) {
-            std::cerr << "No valid benchmarks specified. Exiting." << std::endl;
-            return 1;
+            throw std::runtime_error("No valid benchmarks specified");
         }
     }
 
     benchmark_runner runner{types};
+
     runner.run_microbenches();
-    runner.print_scores();
+    runner.write_scores();
 }