Skip to content

Commit fb26981

Browse files
eramongodberamongodb
authored
CXX-3094 Use Catch2 SKIP() and compact reporter (#1206)
* Scope "Downloading" messages to avoid misleading repeated output * Use compact reporter for Catch2 tests * Use DYNAMIC_SECTION within loops * should_run_client_side_encryption_test -> CLIENT_SIDE_ENCRYPTION_ENABLED_OR_SKIP * server_has_sessions -> SERVER_HAS_SESSIONS_OR_SKIP * Replace WARN + control flow with SKIP * Add --allow-running-no-tests to test flags * Avoid Catch2 issue #1292 * should_skip_spec_test -> CHECK_IF_SKIP_SPEC_TEST * Reduce configure and build command invocations * Add OUTPUT_QUIET to execute_process_and_check_result * Remove redundant build of examples in test.sh * Use foreach() over mongocxx test executables * Consistent plurality for spec tests
1 parent fb04319 commit fb26981

24 files changed

+312
-387
lines changed

‎.evergreen/compile.sh

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,23 +72,24 @@ if [[ -f "${mongoc_prefix:?}/.evergreen/scripts/find-ccache.sh" ]]; then
7272
find_ccache_and_export_vars "$(pwd)" || true
7373
fi
7474

75+
76+
build_targets=()
7577
cmake_build_opts=()
7678
case "${OSTYPE:?}" in
7779
cygwin)
7880
cmake_build_opts+=("/verbosity:minimal")
79-
cmake_examples_target="examples/examples"
81+
build_targets+=(--target ALL_BUILD --target examples/examples)
8082
;;
8183

8284
darwin* | linux*)
83-
cmake_examples_target="examples"
85+
build_targets+=(--target all --target examples)
8486
;;
8587

8688
*)
8789
echo "unrecognized operating system ${OSTYPE:?}" 1>&2
8890
exit 1
8991
;;
9092
esac
91-
: "${cmake_examples_target:?}"
9293

9394
# Create a VERSION_CURRENT file in the build directory to include in the dist tarball.
9495
python ./etc/calc_release_version.py >./build/VERSION_CURRENT
@@ -223,21 +224,23 @@ if [[ -n "${REQUIRED_CXX_STANDARD:-}" ]]; then
223224
cmake_flags+=("-DCMAKE_CXX_STANDARD_REQUIRED=ON")
224225
fi
225226

227+
if [[ "${COMPILE_MACRO_GUARD_TESTS:-"OFF"}" == "ON" ]]; then
228+
cmake_flags+=("-DENABLE_MACRO_GUARD_TESTS=ON")
229+
fi
230+
226231
echo "Configuring with CMake flags: ${cmake_flags[*]}"
227232

228233
"${cmake_binary}" "${cmake_flags[@]}" ..
229234

230235
if [[ "${COMPILE_MACRO_GUARD_TESTS:-"OFF"}" == "ON" ]]; then
231236
# We only need to compile the macro guard tests.
232-
"${cmake_binary}" -DENABLE_MACRO_GUARD_TESTS=ON ..
233237
"${cmake_binary}" --build . --config "${build_type:?}" --target test_bsoncxx_macro_guards test_mongocxx_macro_guards -- "${cmake_build_opts[@]}"
234238
exit # Nothing else to be done.
235239
fi
236240

237241
# Regular build and install routine.
238-
"${cmake_binary}" --build . --config "${build_type:?}" -- "${cmake_build_opts[@]}"
239-
"${cmake_binary}" --build . --config "${build_type:?}" --target install -- "${cmake_build_opts[@]}"
240-
"${cmake_binary}" --build . --config "${build_type:?}" --target "${cmake_examples_target:?}" -- "${cmake_build_opts[@]}"
242+
"${cmake_binary}" --build . --config "${build_type:?}" "${build_targets[@]:?}" -- "${cmake_build_opts[@]}"
243+
"${cmake_binary}" --install . --config "${build_type:?}"
241244

242245
if [[ "${_RUN_DISTCHECK:-}" ]]; then
243246
"${cmake_binary}" --build . --config "${build_type:?}" --target distcheck

‎.evergreen/test.sh

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -262,15 +262,20 @@ else
262262
echo "CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH:?}"
263263
fi
264264

265-
run_test() { "$@"; }
265+
test_args=(
266+
--reporter compact
267+
--allow-running-no-tests
268+
)
269+
270+
run_test() { "$@" "${test_args[@]:?}"; }
266271

267272
if [[ "${TEST_WITH_ASAN:-}" == "ON" || "${TEST_WITH_UBSAN:-}" == "ON" ]]; then
268273
export ASAN_OPTIONS="detect_leaks=1"
269274
export UBSAN_OPTIONS="print_stacktrace=1"
270275
export PATH="/usr/lib/llvm-3.8/bin:${PATH:-}"
271276
elif [[ "${TEST_WITH_VALGRIND:-}" == "ON" ]]; then
272277
run_test() {
273-
valgrind --leak-check=full --track-origins=yes --num-callers=50 --error-exitcode=1 --error-limit=no --read-var-info=yes --suppressions=../etc/memcheck.suppressions "$@"
278+
valgrind --leak-check=full --track-origins=yes --num-callers=50 --error-exitcode=1 --error-limit=no --read-var-info=yes --suppressions=../etc/memcheck.suppressions "$@" "${test_args[@]:?}"
274279
}
275280
fi
276281

@@ -286,11 +291,7 @@ else
286291
run_test ./src/mongocxx/test/test_logging
287292
run_test ./src/mongocxx/test/test_retryable_reads_specs
288293
run_test ./src/mongocxx/test/test_read_write_concern_specs
289-
run_test ./src/mongocxx/test/test_unified_format_spec
290-
291-
echo "Building examples..."
292-
"${cmake_binary:?}" --build . --target examples
293-
echo "Building examples... done."
294+
run_test ./src/mongocxx/test/test_unified_format_specs
294295

295296
# Only run examples if MONGODB_API_VERSION is unset. We do not append
296297
# API version to example clients, so examples will fail when requireApiVersion

‎cmake/FetchCatch2.cmake

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
include(FetchContent)
44

5-
message(STATUS "Downloading Catch2...")
6-
75
function(fetch_catch2)
86
set(fetch_args "")
97
if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.25.0")
@@ -26,6 +24,8 @@ function(fetch_catch2)
2624
FetchContent_GetProperties(EP_Catch2)
2725

2826
if(NOT ep_catch2_POPULATED)
27+
message(STATUS "Downloading Catch2...")
28+
2929
# Avoid Catch2 compile warnings from being treated as errors.
3030
string(REPLACE " -Werror" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
3131
string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
@@ -38,9 +38,9 @@ function(fetch_catch2)
3838
# Catch2 config vars.
3939
set_property(CACHE CATCH_INSTALL_DOCS PROPERTY VALUE OFF)
4040
set_property(CACHE CATCH_INSTALL_EXTRAS PROPERTY VALUE OFF)
41+
42+
message (STATUS "Downloading Catch2... done.")
4143
endif()
4244
endfunction()
4345

4446
fetch_catch2()
45-
46-
message (STATUS "Downloading Catch2... done.")

‎cmake/make_dist/MakeDistFiles.cmake

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ function (EXECUTE_PROCESS_AND_CHECK_RESULT)
2727
${VARS_COMMAND}
2828
WORKING_DIRECTORY ${VARS_WORKING_DIRECTORY}
2929
RESULT_VARIABLE RESULT
30+
OUTPUT_QUIET
3031
)
3132
if (NOT "${RESULT}" STREQUAL "0")
3233
message (FATAL_ERROR ${VARS_ERROR_MSG})

‎src/bsoncxx/test/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ target_link_libraries(bsoncxx_test_properties_with_main INTERFACE bsoncxx::test_
7070
add_library(bsoncxx::test_properties_with_main ALIAS bsoncxx_test_properties_with_main)
7171

7272
target_link_libraries (test_bson PRIVATE bsoncxx::test_properties_with_main)
73-
add_test(NAME bson COMMAND test_bson)
73+
74+
add_test(NAME bson COMMAND test_bson --reporter compact --allow-running-no-tests)
7475

7576
# Generate test to ensure macro guards behave properly.
7677
if(ENABLE_MACRO_GUARD_TESTS)

‎src/mongocxx/test/CMakeLists.txt

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,8 @@ target_link_libraries(test_read_write_concern_specs PRIVATE spec_test_common cli
135135
add_executable(test_mongohouse_specs spec/mongohouse.cpp)
136136
target_link_libraries(test_mongohouse_specs PRIVATE spec_test_common client_helpers)
137137

138-
add_executable(test_unified_format_spec spec/unified_tests/operations.cpp spec/unified_tests/runner.cpp)
139-
target_link_libraries(test_unified_format_spec PRIVATE spec_test_common client_helpers)
138+
add_executable(test_unified_format_specs spec/unified_tests/operations.cpp spec/unified_tests/runner.cpp)
139+
target_link_libraries(test_unified_format_specs PRIVATE spec_test_common client_helpers)
140140

141141
add_executable(test_versioned_api versioned_api.cpp)
142142
target_link_libraries(test_versioned_api PRIVATE spec_test_common client_helpers)
@@ -183,7 +183,7 @@ set_property(
183183
test_read_write_concern_specs
184184
test_retryable_reads_specs
185185
test_transactions_specs
186-
test_unified_format_spec
186+
test_unified_format_specs
187187
test_versioned_api
188188
APPEND PROPERTY LINK_LIBRARIES
189189
mongocxx::test_properties_with_main
@@ -206,23 +206,27 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
206206
find_package(Threads REQUIRED)
207207
target_link_libraries(test_driver PRIVATE Threads::Threads)
208208

209-
add_test(NAME driver COMMAND test_driver)
210-
add_test(NAME logging COMMAND test_logging)
211-
add_test(NAME instance COMMAND test_instance)
212-
add_test(NAME crud_specs COMMAND test_crud_specs)
213-
add_test(NAME gridfs_specs COMMAND test_gridfs_specs)
214-
add_test(NAME client_side_encryption_specs COMMAND test_client_side_encryption_specs)
215-
add_test(NAME command_monitoring_specs COMMAND test_command_monitoring_specs)
216-
add_test(NAME transactions_specs COMMAND test_transactions_specs)
217-
add_test(NAME retryable_reads_spec COMMAND test_retryable_reads_specs)
218-
add_test(NAME read_write_concern_specs COMMAND test_read_write_concern_specs)
219-
add_test(NAME unified_format_spec COMMAND test_unified_format_spec)
220-
add_test(NAME versioned_api COMMAND test_versioned_api)
209+
foreach(test_name
210+
client_side_encryption_specs
211+
command_monitoring_specs
212+
crud_specs
213+
driver
214+
gridfs_specs
215+
instance
216+
logging
217+
read_write_concern_specs
218+
retryable_reads_specs
219+
transactions_specs
220+
unified_format_specs
221+
versioned_api
222+
)
223+
add_test(NAME ${test_name} COMMAND test_${test_name} --reporter compact --allow-running-no-tests)
224+
endforeach()
221225

222226
# Adding this as a test will run it as part of the RUN_TESTS command in MSVC.
223227
# Do not add, since we only test mongohouse on Linux.
224228
if(0)
225-
add_test(mongohouse_specs NAME test_mongohouse_specs COMMAND)
229+
add_test(NAME mongohouse_specs COMMAND test_mongohouse_specs ${test_args})
226230
endif()
227231

228232
set_property(TEST crud_specs APPEND PROPERTY ENVIRONMENT
@@ -251,15 +255,15 @@ set_property(TEST transactions_specs APPEND PROPERTY ENVIRONMENT
251255
"WITH_TRANSACTION_TESTS_PATH=${DATA_SOURCE_DIR}/with_transaction"
252256
)
253257

254-
set_property(TEST retryable_reads_spec APPEND PROPERTY ENVIRONMENT
258+
set_property(TEST retryable_reads_specs APPEND PROPERTY ENVIRONMENT
255259
"RETRYABLE_READS_LEGACY_TESTS_PATH=${DATA_SOURCE_DIR}/retryable-reads/legacy"
256260
)
257261

258262
set_property(TEST read_write_concern_specs APPEND PROPERTY ENVIRONMENT
259263
"READ_WRITE_CONCERN_OPERATION_TESTS_PATH=${DATA_SOURCE_DIR}/read-write-concern/operation"
260264
)
261265

262-
set_property(TEST unified_format_spec APPEND PROPERTY ENVIRONMENT
266+
set_property(TEST unified_format_specs APPEND PROPERTY ENVIRONMENT
263267
"CHANGE_STREAMS_UNIFIED_TESTS_PATH=${DATA_SOURCE_DIR}/change-streams/unified"
264268
"CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS_PATH=${DATA_SOURCE_DIR}/client_side_encryption/unified"
265269
"COLLECTION_MANAGEMENT_TESTS_PATH=${DATA_SOURCE_DIR}/collection-management"

‎src/mongocxx/test/change_streams.cpp

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ TEST_CASE("Change stream options") {
9595
client mongodb_client{uri{}, test_util::add_test_server_api()};
9696

9797
if (!test_util::is_replica_set(mongodb_client)) {
98-
WARN("skip: change streams require replica set");
99-
return;
98+
SKIP("change streams require replica set");
10099
}
101100

102101
SECTION("Error if both resumeAfter and startAfter are set") {
@@ -118,8 +117,7 @@ TEST_CASE("Spec Prose Tests") {
118117
client client{uri{}, test_util::add_test_server_api()};
119118

120119
if (!test_util::is_replica_set(client)) {
121-
WARN("skip: change streams require replica set");
122-
return;
120+
SKIP("change streams require replica set");
123121
}
124122

125123
auto db = client["db"];
@@ -387,8 +385,7 @@ TEST_CASE("Create streams.events and assert we can read a single event", "[min36
387385
instance::current();
388386
client mongodb_client{uri{}, test_util::add_test_server_api()};
389387
if (!test_util::is_replica_set(mongodb_client)) {
390-
WARN("skip: change streams require replica set");
391-
return;
388+
SKIP("change streams require replica set");
392389
}
393390

394391
collection events = mongodb_client["streams"]["events"];
@@ -408,8 +405,7 @@ TEST_CASE("Give an invalid pipeline", "[min36]") {
408405
instance::current();
409406
client mongodb_client{uri{}, test_util::add_test_server_api()};
410407
if (!test_util::is_replica_set(mongodb_client)) {
411-
WARN("skip: change streams require replica set");
412-
return;
408+
SKIP("change streams require replica set");
413409
}
414410

415411
options::change_stream options{};
@@ -439,8 +435,7 @@ TEST_CASE("Documentation Examples", "[min36]") {
439435
mongocxx::pool pool{uri{}, options::pool(test_util::add_test_server_api())};
440436
auto mongodb_client = pool.acquire();
441437
if (!test_util::is_replica_set(*mongodb_client)) {
442-
WARN("skip: change streams require replica set");
443-
return;
438+
SKIP("change streams require replica set");
444439
}
445440

446441
collection events = (*mongodb_client)["streams"]["events"];
@@ -546,8 +541,7 @@ TEST_CASE("Watch 2 collections", "[min36]") {
546541
instance::current();
547542
client mongodb_client{uri{}, test_util::add_test_server_api()};
548543
if (!test_util::is_replica_set(mongodb_client)) {
549-
WARN("skip: change streams require replica set");
550-
return;
544+
SKIP("change streams require replica set");
551545
}
552546

553547
options::change_stream options{};
@@ -594,8 +588,7 @@ TEST_CASE("Watch a Collection", "[min36]") {
594588
instance::current();
595589
client mongodb_client{uri{}, test_util::add_test_server_api()};
596590
if (!test_util::is_replica_set(mongodb_client)) {
597-
WARN("skip: change streams require replica set");
598-
return;
591+
SKIP("change streams require replica set");
599592
}
600593

601594
options::change_stream options{};

‎src/mongocxx/test/client_helpers.cpp

Lines changed: 28 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -492,59 +492,53 @@ void check_outcome_collection(mongocxx::collection* coll, bsoncxx::document::vie
492492
REQUIRE(begin(actual) == end(actual));
493493
}
494494

495-
bool server_has_sessions(const client& conn) {
495+
bool server_has_sessions_impl(const client& conn) {
496496
auto result = get_is_master(conn);
497497
auto result_view = result.view();
498498

499499
if (result_view["logicalSessionTimeoutMinutes"]) {
500500
return true;
501501
}
502502

503-
WARN("skip: server does not support sessions");
504503
return false;
505504
}
506505

507-
bool should_run_client_side_encryption_test(void) {
508-
#ifndef MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION
509-
WARN("linked libmongoc does not support client side encryption - skipping tests");
510-
return false;
511-
#endif
512-
513-
std::vector<const char*> vars{
514-
"MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY",
515-
"MONGOCXX_TEST_AWS_ACCESS_KEY_ID",
516-
"MONGOCXX_TEST_AZURE_TENANT_ID",
517-
"MONGOCXX_TEST_AZURE_CLIENT_ID",
518-
"MONGOCXX_TEST_AZURE_CLIENT_SECRET",
519-
"MONGOCXX_TEST_CSFLE_TLS_CA_FILE",
520-
"MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE",
521-
"MONGOCXX_TEST_GCP_EMAIL",
522-
"MONGOCXX_TEST_GCP_PRIVATEKEY",
523-
};
524-
525-
std::ostringstream os;
526-
os << "Please set environment variables to enable client side encryption tests:\n";
527-
std::copy(std::begin(vars), std::end(vars), std::ostream_iterator<const char*>(os, "\n"));
506+
#if defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION)
528507

529-
if (std::none_of(std::begin(vars), std::end(vars), std::getenv)) {
530-
os << "Skipping client side encryption tests.\n";
508+
cseeos_result client_side_encryption_enabled_or_skip_impl() {
509+
static const cseeos_result result = [] {
510+
std::vector<const char*> vars{
511+
"MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY",
512+
"MONGOCXX_TEST_AWS_ACCESS_KEY_ID",
513+
"MONGOCXX_TEST_AZURE_TENANT_ID",
514+
"MONGOCXX_TEST_AZURE_CLIENT_ID",
515+
"MONGOCXX_TEST_AZURE_CLIENT_SECRET",
516+
"MONGOCXX_TEST_CSFLE_TLS_CA_FILE",
517+
"MONGOCXX_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE",
518+
"MONGOCXX_TEST_GCP_EMAIL",
519+
"MONGOCXX_TEST_GCP_PRIVATEKEY",
520+
};
531521

532-
WARN(os.str());
522+
const auto is_set = [](const char* var) -> bool { return std::getenv(var) != nullptr; };
533523

534-
return false;
535-
}
524+
const auto count = std::count_if(vars.begin(), vars.end(), is_set);
536525

537-
if (!std::all_of(std::begin(vars), std::end(vars), std::getenv)) {
538-
os << "Failing client side encryption tests (some environment variables were not set).\n";
526+
if (count == 0) {
527+
return cseeos_result::skip;
528+
}
539529

540-
FAIL(os.str());
530+
if (static_cast<std::size_t>(count) < vars.size()) {
531+
return cseeos_result::fail;
532+
}
541533

542-
return false;
543-
}
534+
return cseeos_result::enable;
535+
}();
544536

545-
return true;
537+
return result;
546538
}
547539

540+
#endif // defined(MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION)
541+
548542
std::string getenv_or_fail(const std::string env_name) {
549543
auto val = std::getenv(env_name.c_str());
550544
if (!val) {

0 commit comments

Comments
 (0)