Merge pull request #300 from ldorau/Unify_benchmarks_CI_workflows_for_Linux_and_Windows

Unify benchmarks' CI workflows for Linux and Windows

Author: lukaszstolarczuk

.github/workflows/basic.yml

@@ -158,7 +158,7 @@ jobs:
 
     - name: Run tests
       working-directory: ${{env.BUILD_DIR}}
-      run: ctest --output-on-failure
+      run: ctest --output-on-failure --test-dir test
 
     - name: Test make install
       # Run only when the example is built
@@ -246,7 +246,7 @@ jobs:
 
     - name: Run tests
       working-directory: ${{env.BUILD_DIR}}
-      run: ctest -C ${{matrix.build_type}} --output-on-failure
+      run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
 
   macos-build:
     name: MacOS

.github/workflows/benchmarks.yml

@@ -7,126 +7,72 @@ permissions:
   contents: read
 
 jobs:
-  benchmarks-ubuntu:
-    name: Ubuntu
+  benchmarks:
+    name: Benchmarks
+    env:
+      BUILD_DIR : "${{github.workspace}}/build/"
+      VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows"
     strategy:
       matrix:
-        os: ['ubuntu-22.04']
-        build_type: [Release]
-        compiler: [{c: gcc, cxx: g++}]
-        shared_library: ['ON', 'OFF']
+        include:
+          - os: ubuntu-latest
+            pool_jemalloc: 'ON'
+
+          - os: windows-latest
+            pool_jemalloc: 'OFF'
+
     runs-on: ${{matrix.os}}
 
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Install apt packages
+        if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get update
           sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
 
+      - name: Initialize vcpkg
+        if: matrix.os == 'windows-latest'
+        uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
+        with:
+          vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
+          vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
+          vcpkgJsonGlob: '**/vcpkg.json'
+
+      - name: Install vcpkg packages
+        if: matrix.os == 'windows-latest'
+        run: vcpkg install
+        shell: pwsh # Specifies PowerShell as the shell for running the script.
+
       - name: Configure build
         run: >
           cmake
-          -B ${{github.workspace}}/build
-          -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
-          -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
-          -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
-          -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
+          -B ${{env.BUILD_DIR}}
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
+          -DUMF_BUILD_SHARED_LIBRARY=ON
           -DUMF_BUILD_BENCHMARKS=ON
           -DUMF_BUILD_BENCHMARKS_MT=ON
           -DUMF_BUILD_TESTS=OFF
           -DUMF_FORMAT_CODE_STYLE=OFF
           -DUMF_DEVELOPER_MODE=OFF
           -DUMF_BUILD_OS_MEMORY_PROVIDER=ON
-          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
+          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{matrix.pool_jemalloc}}
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
           -DUMF_ENABLE_POOL_TRACKING=OFF
           -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
-      - name: Build UMF
-        run: cmake --build ${{github.workspace}}/build -j $(nproc)
-
-      - name: Run benchmarks
-        working-directory: ${{github.workspace}}/build
-        # The benchmark of TBB pool manager fails very often with the
-        # "confidence interval exceeds maximum permitted 2.5%" error.
-        # Do not treat that as a failure in CI.
-        run: |
-          export LOG=/tmp/ubench.log
-          if ! ./benchmark/ubench 2>/dev/null > $LOG; then \
-            cat $LOG; \
-            if ! grep -q -e "exceeds maximum permitted 2.5" $LOG; then \
-              echo "[  FAILED  ] The CI benchmark job FAILED."; \
-              exit 1; \
-            fi; \
-          fi
-          cat $LOG
-          echo "[  PASSED  ] The CI benchmark job PASSED."
+      - name: Build UMF on Linux
+        if: matrix.os == 'ubuntu-latest'
+        run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
 
-      - name: Run MT benchmarks
-        working-directory: ${{github.workspace}}/build
-        run: ./benchmark/multithread_bench
+      - name: Build UMF on Windows
+        if: matrix.os == 'windows-latest'
+        run: cmake --build ${{env.BUILD_DIR}} --config Release -j $Env:NUMBER_OF_PROCESSORS
 
-  benchmarks-windows:
-    name: Windows
-    env:
-      BUILD_DIR : "${{github.workspace}}/build/"
-      VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows"
-      VCPKG_PATH_BIN: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows/bin;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows/bin"
-    strategy:
-      fail-fast: false
-      matrix:
-        build_type: [Release]
-        compiler: [{c: cl, cxx: cl}]
-        shared_library: ['ON', 'OFF']
-    runs-on: 'windows-2022'
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-    - name: Initialize vcpkg
-      uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
-      with:
-        vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
-        vcpkgDirectory: ${{github.workspace}}/build/vcpkg
-        vcpkgJsonGlob: '**/vcpkg.json'
-
-    - name: Install dependencies
-      run: vcpkg install
-      shell: pwsh # Specifies PowerShell as the shell for running the script.
-
-    - name: Configure build
-      run: >
-        cmake
-        -B ${{env.BUILD_DIR}}
-        -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
-        -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
-        -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
-        -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
-        -DUMF_BUILD_BENCHMARKS=ON
-        -DUMF_BUILD_BENCHMARKS_MT=OFF
-        -DUMF_BUILD_TESTS=OFF
-        -DUMF_FORMAT_CODE_STYLE=OFF
-        -DUMF_DEVELOPER_MODE=OFF
-        -DUMF_BUILD_OS_MEMORY_PROVIDER=ON
-        -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
-        -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF
-        -DUMF_BUILD_LIBUMF_POOL_SCALABLE=OFF
-        -DUMF_ENABLE_POOL_TRACKING=OFF
-
-    - name: Build UMF
-      run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS
-
-    - name: Run benchmarks
-      working-directory: ${{env.BUILD_DIR}}
-      shell: pwsh # Specifies PowerShell as the shell for running the script.
-      run: |
-        # add path to umf.dll
-        $env:PATH += ";${{env.BUILD_DIR}}\bin\${{matrix.build_type}}"
-        # add path to DLL directories
-        $env:PATH += ";${{env.VCPKG_PATH_BIN}}"
-        .\benchmark\${{matrix.build_type}}\ubench.exe
+      - name: Run benchmarks
+        working-directory: ${{env.BUILD_DIR}}
+        run: ctest -V --test-dir benchmark -C Release

benchmark/CMakeLists.txt

@@ -2,92 +2,102 @@
 # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-include(FindThreads)
-
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
 	message(WARNING "The benchmarks SHOULD NOT be run in the Debug build type!")
 endif()
 
+if (UMF_BUILD_BENCHMARKS_MT)
+	include(FindThreads)
+endif()
+
+function(add_umf_benchmark)
+    # NAME - a name of the benchmark
+    # SRCS - source files
+    # LIBS - libraries to be linked with
+    # LIBDIRS - directories of libraries to be linked with
+    set(oneValueArgs NAME)
+    set(multiValueArgs SRCS LIBS LIBDIRS)
+    cmake_parse_arguments(ARG "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    set(BENCH_NAME umf-bench-${ARG_NAME})
+
+    set(BENCH_LIBS umf ${ARG_LIBS})
+
+    add_umf_executable(NAME ${BENCH_NAME} SRCS ${ARG_SRCS} LIBS ${BENCH_LIBS})
+
+    target_include_directories(${BENCH_NAME} PRIVATE
+        ${UMF_CMAKE_SOURCE_DIR}/include
+        ${UMF_CMAKE_SOURCE_DIR}/src/utils)
+
+    target_link_directories(${BENCH_NAME} PRIVATE ${ARG_LIBDIRS})
+
+    add_test(NAME ${BENCH_NAME}
+        COMMAND ${BENCH_NAME}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+    # Benchmark passes if it prints "PASSED" in the output,
+    # because ubench of scalable pool fails if the confidence interval
+    # exceeds maximum permitted 2.5%.
+    set_tests_properties(${BENCH_NAME} PROPERTIES
+        LABELS "benchmark"
+        PASS_REGULAR_EXPRESSION "PASSED")
+
+    if(WINDOWS)
+        # append PATH to DLLs
+        set(DLL_PATH_LIST "PATH=path_list_append:../bin/$<CONFIG>")
+        if(LIBHWLOC_LIBRARY_DIRS)
+            set(DLL_PATH_LIST "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin")
+        endif()
+        if(TBB_LIBRARY_DIRS)
+            set(DLL_PATH_LIST "${DLL_PATH_LIST};PATH=path_list_append:${TBB_LIBRARY_DIRS}/../bin")
+        endif()
+        set_property(TEST ${BENCH_NAME} PROPERTY ENVIRONMENT_MODIFICATION "${DLL_PATH_LIST}")
+    endif()
+
+    if (UMF_BUILD_OS_MEMORY_PROVIDER)
+        target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
+    endif()
+    if (UMF_BUILD_LIBUMF_POOL_DISJOINT)
+        target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
+    endif()
+    if (UMF_BUILD_LIBUMF_POOL_JEMALLOC)
+        target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
+    endif()
+    if (UMF_BUILD_LIBUMF_POOL_SCALABLE)
+        target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
+    endif()
+endfunction()
+
+# optional libraries
 if(UMF_BUILD_LIBUMF_POOL_DISJOINT)
 	set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool)
 endif()
-
 if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
 	set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool)
 endif()
-
 if(UMF_BUILD_LIBUMF_POOL_SCALABLE)
 	set(LIBS_OPTIONAL ${LIBS_OPTIONAL} scalable_pool)
 endif()
-
-add_executable(ubench ubench.c)
-
 if(LINUX)
-	set(LIBS_LINUX m)
+	set(LIBS_OPTIONAL ${LIBS_OPTIONAL} m)
 endif()
-
-add_dependencies(ubench
-	umf
-	${LIBS_OPTIONAL})
-
 if(UMF_BUILD_OS_MEMORY_PROVIDER)
     set(LIBS_OPTIONAL ${LIBS_OPTIONAL} ${LIBHWLOC_LIBRARIES})
     set(LIB_DIRS_OPTIONAL ${LIB_DIRS_OPTIONAL} ${LIBHWLOC_LIBRARY_DIRS})
 endif()
 
-target_include_directories(ubench PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include/ ${UMF_CMAKE_SOURCE_DIR}/src/utils/)
+# BENCHMARKS
 
-target_link_directories(ubench PRIVATE ${LIB_DIRS_OPTIONAL})
-
-target_link_libraries(ubench
-	umf
-	${LIBS_OPTIONAL}
-	${CMAKE_THREAD_LIBS_INIT}
-	${LIBS_LINUX})
+add_umf_benchmark(
+    NAME ubench
+    SRCS ubench.c
+    LIBS ${LIBS_OPTIONAL}
+    LIBDIRS ${LIB_DIRS_OPTIONAL})
 
 if (UMF_BUILD_BENCHMARKS_MT)
-	add_executable(multithread_bench multithread.cpp)
-	target_link_directories(multithread_bench PRIVATE ${LIB_DIRS_OPTIONAL})
-	target_link_libraries(multithread_bench
-		umf
-		${LIBS_OPTIONAL}
-		${CMAKE_THREAD_LIBS_INIT}
-		${LIBS_LINUX})
-	target_include_directories(multithread_bench PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include/)
-endif()
-
-if (UMF_BUILD_OS_MEMORY_PROVIDER)
-	target_compile_definitions(ubench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
-
-	if (UMF_BUILD_BENCHMARKS_MT)
-		target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
-	endif()
-endif()
-
-if (UMF_BUILD_OS_MEMORY_PROVIDER)
-	target_compile_definitions(ubench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
-endif()
-
-if (UMF_BUILD_LIBUMF_POOL_DISJOINT)
-	target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
-
-	if (UMF_BUILD_BENCHMARKS_MT)
-		target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
-	endif()
-endif()
-
-if (UMF_BUILD_LIBUMF_POOL_JEMALLOC)
-	target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
-
-	if (UMF_BUILD_BENCHMARKS_MT)
-		target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
-	endif()
-endif()
-
-if (UMF_BUILD_LIBUMF_POOL_SCALABLE)
-	target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
-
-	if (UMF_BUILD_BENCHMARKS_MT)
-		target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
-	endif()
+    add_umf_benchmark(
+        NAME multithreaded
+        SRCS multithread.cpp
+        LIBS ${LIBS_OPTIONAL} ${CMAKE_THREAD_LIBS_INIT}
+        LIBDIRS ${LIB_DIRS_OPTIONAL})
 endif()

benchmark/multithread.cpp

@@ -65,15 +65,15 @@ static void mt_alloc_free(poolCreateExtParams params,
     auto values = umf_bench::measure<std::chrono::milliseconds>(
         bench.n_repeats, bench.n_threads,
         [&, pool = pool.get()](auto thread_id) {
-            for (int i = 0; i < bench.n_iterations; i++) {
+            for (size_t i = 0; i < bench.n_iterations; i++) {
                 allocs[thread_id].push_back(
                     umfPoolMalloc(pool, bench.alloc_size));
                 if (!allocs[thread_id].back()) {
                     numFailures[thread_id]++;
                 }
             }
 
-            for (int i = 0; i < bench.n_iterations; i++) {
+            for (size_t i = 0; i < bench.n_iterations; i++) {
                 umfPoolFree(pool, allocs[thread_id][i]);
             }
 
@@ -131,5 +131,8 @@ int main() {
     std::cout << "skipping disjoint_pool mt_alloc_free" << std::endl;
 #endif
 
+    // ctest looks for "PASSED" in the output
+    std::cout << "PASSED" << std::endl;
+
     return 0;
 }