Run microbenchmarks in Evergreen

Author: adriandole

.mci.yml

@@ -360,6 +360,16 @@ functions:
                   CXX: ${cxx_compiler}
               script: .evergreen/compile.sh
 
+    "compile_benchmarks":
+      - command: shell.exec
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: |
+            cd build
+            cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(pwd)/../mongoc" -DCMAKE_CXX_STANDARD=20 ..
+            make microbenchmarks -j64
+
     "test":
         - command: shell.exec
           params:
@@ -510,6 +520,30 @@ functions:
             echo "Building Red Hat UBI Docker image"
             make -C extras/docker/redhat-ubi-9.3 nocachebuild test
 
+    "download benchmark data":
+      - command: shell.exec
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: |
+            ./etc/microbenchmark-test-data.sh
+
+    "run benchmarks":
+      - command: shell.exec
+        params:
+          shell: bash
+          working_dir: "mongo-cxx-driver"
+          script: |
+            echo "Starting benchmarks"
+            ./build/benchmark/microbenchmarks all
+
+    "send benchmarks":
+      - command: perf.send
+        params:
+          name: perf
+          file: mongo-cxx-driver/results.json
+
+
 #######################################
 #              Post Task              #
 #######################################
@@ -620,6 +654,16 @@ tasks:
         vars:
             ENABLE_TESTS: OFF
 
+    - name: compile_and_run_benchmarks
+      commands:
+      - func: "setup"
+      - func: "start_mongod"
+      - func: "fetch_c_driver_source"
+      - func: "compile_benchmarks"
+      - func: "download benchmark data"
+      - func: "run benchmarks"
+      - func: "send benchmarks"
+
     - name: compile_macro_guard_tests
       commands:
       - func: "setup"
@@ -1303,6 +1347,15 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
 
+    - name: benchmarks-rhel9
+      display_name: "Benchmarks (RHEL 9.2)"
+      expansions:
+          mongodb_version: "v6.0-perf"
+      run_on:
+          - rhel90-dbx-perf-large
+      tasks:
+          - name: compile_and_run_benchmarks
+
     - name: arm-rhel9-release-latest
       display_name: "arm64 RHEL 9 Release (MongoDB Latest)"
       expansions:

benchmark/benchmark_runner.cpp

@@ -14,6 +14,14 @@
 
 #include "benchmark_runner.hpp"
 
+#include <chrono>
+#include <cstdint>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+
 #include "bson/bson_encoding.hpp"
 #include "multi_doc/bulk_insert.hpp"
 #include "multi_doc/find_many.hpp"
@@ -26,7 +34,13 @@
 #include "single_doc/find_one_by_id.hpp"
 #include "single_doc/insert_one.hpp"
 #include "single_doc/run_command.hpp"
+#include <bsoncxx/builder/basic/array.hpp>
+#include <bsoncxx/builder/basic/document.hpp>
+#include <bsoncxx/builder/basic/kvp.hpp>
+#include <bsoncxx/builder/basic/sub_document.hpp>
+#include <bsoncxx/json.hpp>
 #include <bsoncxx/stdx/make_unique.hpp>
+#include <bsoncxx/types.hpp>
 
 namespace benchmark {
 
@@ -47,26 +61,27 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
     _microbenches.push_back(make_unique<run_command>());
     _microbenches.push_back(make_unique<find_one_by_id>("single_and_multi_document/tweet.json"));
     _microbenches.push_back(make_unique<insert_one>(
-        "TestSmallDocInsertOne", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+        "TestSmallDocInsertOne", 2.75, iterations, "single_and_multi_document/small_doc.json"));
     _microbenches.push_back(make_unique<insert_one>(
         "TestLargeDocInsertOne", 27.31, 10, "single_and_multi_document/large_doc.json"));
 
     // Multi doc microbenchmarks
     _microbenches.push_back(make_unique<find_many>("single_and_multi_document/tweet.json"));
     _microbenches.push_back(make_unique<bulk_insert>(
-        "TestSmallDocBulkInsert", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+        "TestSmallDocBulkInsert", 2.75, iterations, "single_and_multi_document/small_doc.json"));
     _microbenches.push_back(make_unique<bulk_insert>(
         "TestLargeDocBulkInsert", 27.31, 10, "single_and_multi_document/large_doc.json"));
-    _microbenches.push_back(
-        make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
-    _microbenches.push_back(
-        make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
+    // CXX-2794: Disable GridFS benchmarks due to long runtime
+    // _microbenches.push_back(
+    //     make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
+    // _microbenches.push_back(
+    //     make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
 
     // Parallel microbenchmarks
     _microbenches.push_back(make_unique<json_multi_import>("parallel/ldjson_multi"));
     _microbenches.push_back(make_unique<json_multi_export>("parallel/ldjson_multi"));
-    _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
-    _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
+    // _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
+    // _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
 
     // Need to remove some
     if (!_types.empty()) {
@@ -89,8 +104,6 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
 }
 
 void benchmark_runner::run_microbenches() {
-    mongocxx::instance instance{};
-
     for (std::unique_ptr<microbench>& bench : _microbenches) {
         std::cout << "Starting " << bench->get_name() << "..." << std::endl;
 
@@ -145,20 +158,50 @@ double benchmark_runner::calculate_driver_bench_score() {
     return (calculate_read_bench_score() + calculate_write_bench_score()) / 2.0;
 }
 
-void benchmark_runner::print_scores() {
+void benchmark_runner::write_scores(
+    const std::chrono::time_point<std::chrono::system_clock> start_time) {
     double read = -1;
     double write = -1;
+    const auto end_time = std::chrono::system_clock::now();
+
+    using namespace bsoncxx;
+    using builder::basic::sub_document;
+
+    auto doc = builder::basic::document{};
+    doc.append(kvp("info", [](sub_document subdoc) {
+        subdoc.append(kvp("test_name", "C++ microbenchmarks"));
+    }));
+
+    auto write_time =
+        [](const std::chrono::time_point<std::chrono::system_clock> t) -> std::string {
+        std::time_t t1 = std::chrono::system_clock::to_time_t(t);
+        std::ostringstream oss;
+        oss << std::put_time(std::gmtime(&t1), "%Y-%m-%dT%H:%M:%S") << "+00:00";
+        return oss.str();
+    };
+    doc.append(kvp("created_at", write_time(start_time)));
+    doc.append(kvp("completed_at", write_time(end_time)));
+    doc.append(kvp("artifacts", builder::basic::make_array()));
+
+    auto metrics_array = builder::basic::array{};
+    std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
 
     std::cout << "Individual microbenchmark scores:" << std::endl << "===========" << std::endl;
     for (auto&& bench : _microbenches) {
         auto& score = bench->get_results();
+        const auto bench_time = static_cast<double>(score.get_percentile(50).count()) / 1000.0;
 
-        std::cout << bench->get_name() << ": "
-                  << static_cast<double>(score.get_percentile(50).count()) / 1000.0
-                  << " second(s) | " << score.get_score() << " MB/s" << std::endl;
-    }
+        std::cout << bench->get_name() << ": " << bench_time << " seconds | " << score.get_score()
+                  << " MB/s" << std::endl;
 
-    std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+        auto metric_doc = builder::basic::document{};
+        metric_doc.append(kvp("name", bench->get_name()));
+        metric_doc.append(kvp("type", "THROUGHPUT"));
+        metric_doc.append(kvp("value", score.get_score()));
+        metrics_array.append(metric_doc);
+    }
+    doc.append(kvp("metrics", metrics_array));
+    doc.append(kvp("sub_tests", builder::basic::make_array()));
 
     auto print_comp = [this, &read, &write](benchmark_type type) {
         double avg = calculate_average(type);
@@ -169,7 +212,7 @@ void benchmark_runner::print_scores() {
             write = avg;
         }
 
-        std::cout << type_names[type] << " " << avg << " MB/s" << std::endl;
+        std::cout << type_names.at(type) << " " << avg << " MB/s" << std::endl;
     };
 
     if (!_types.empty()) {
@@ -185,5 +228,8 @@ void benchmark_runner::print_scores() {
     if (read > 0 && write > 0) {
         std::cout << "DriverBench: " << (read + write) / 2.0 << " MB/s" << std::endl;
     }
+
+    std::ofstream os{"results.json"};
+    os << '[' << bsoncxx::to_json(doc.view()) << ']';
 }
 }  // namespace benchmark

benchmark/benchmark_runner.hpp

@@ -14,10 +14,11 @@
 
 #pragma once
 
-#include <bsoncxx/stdx/optional.hpp>
-#include <mongocxx/instance.hpp>
+#include <chrono>
 
 #include "microbench.hpp"
+#include <bsoncxx/stdx/optional.hpp>
+#include <mongocxx/instance.hpp>
 
 namespace benchmark {
 
@@ -27,7 +28,7 @@ class benchmark_runner {
 
     void run_microbenches();
 
-    void print_scores();
+    void write_scores(const std::chrono::time_point<std::chrono::system_clock> start_time);
 
     double calculate_bson_bench_score();
 

benchmark/bson/bson_decoding.hpp

@@ -42,7 +42,7 @@ void bson_decoding::setup() {
 }
 
 void bson_decoding::task() {
-    for (std::uint32_t i = 0; i < 10000; i++) {
+    for (std::uint32_t i = 0; i < iterations; i++) {
         // TODO CXX-1241: call bson_as_extended json on _json.
     }
 }

benchmark/bson/bson_encoding.hpp

@@ -16,11 +16,10 @@
 
 #include <iostream>
 
+#include "../microbench.hpp"
 #include <bsoncxx/json.hpp>
 #include <bsoncxx/types.hpp>
 
-#include "../microbench.hpp"
-
 namespace benchmark {
 
 class bson_encoding : public microbench {
@@ -58,7 +57,7 @@ void visit_document(bsoncxx::document::view doc) {
 
 // Mirroring mongo-c-driver's interpretation of the spec.
 void bson_encoding::task() {
-    for (std::uint32_t i = 0; i < 10000; i++) {
+    for (std::uint32_t i = 0; i < iterations; i++) {
         visit_document(_doc->view());
     }
 }

benchmark/main.cpp

@@ -12,24 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <chrono>
 #include <iostream>
 
 #include "benchmark_runner.hpp"
+#include <bsoncxx/stdx/string_view.hpp>
+#include <mongocxx/instance.hpp>
 
 using namespace benchmark;
 
 int main(int argc, char* argv[]) {
+    mongocxx::instance instance{};
     std::set<benchmark_type> types;
 
     if (argc > 1) {
-        for (int x = 1; x < argc; ++x) {
-            std::string type{argv[x]};
-            auto it = names_types.find(type);
-
-            if (it != names_types.end()) {
-                types.insert(it->second);
-            } else {
-                std::cerr << "Invalid benchmark: " << type << std::endl;
+        if (bsoncxx::stdx::string_view(argv[1]) == "all") {
+            for (const auto& [name, type] : names_types) {
+                types.insert(type);
+            }
+        } else {
+            for (int x = 1; x < argc; ++x) {
+                std::string type{argv[x]};
+                auto it = names_types.find(type);
+
+                if (it != names_types.end()) {
+                    types.insert(it->second);
+                } else {
+                    std::cerr << "Invalid benchmark: " << type << std::endl;
+                }
             }
         }
 
@@ -40,6 +50,8 @@ int main(int argc, char* argv[]) {
     }
 
     benchmark_runner runner{types};
+
+    const auto start_time = std::chrono::system_clock::now();
     runner.run_microbenches();
-    runner.print_scores();
+    runner.write_scores(start_time);
 }

benchmark/microbench.cpp

@@ -23,7 +23,7 @@ namespace benchmark {
 
 bool finished_running(const std::chrono::duration<std::uint32_t, std::milli>& curr_time,
                       std::uint32_t iter) {
-    return (curr_time > maxtime || (curr_time > mintime && iter > MAX_ITER));
+    return (curr_time > maxtime || (curr_time > mintime && iter > max_iter));
 }
 
 void microbench::run() {