Skip to content

Commit 6169483

Browse files
ldorauldorau
committed
Unify benchmarks' CI workflows for Linux and Windows
Unify benchmarks' CI workflows for Linux and Windows. Use ctest to run benchmarks. A benchmark passes if it prints "PASSED" in the output, because ubench of scalable pool fails if the confidence interval exceeds maximum permitted 2.5%. Signed-off-by: Lukasz Dorau <[email protected]>
1 parent 38173d5 commit 6169483

File tree

3 files changed

+120
-156
lines changed

3 files changed

+120
-156
lines changed

‎.github/workflows/basic.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ jobs:
158158

159159
- name: Run tests
160160
working-directory: ${{env.BUILD_DIR}}
161-
run: ctest --output-on-failure
161+
run: ctest --output-on-failure --test-dir test
162162

163163
- name: Test make install
164164
# Run only when the example is built
@@ -245,7 +245,7 @@ jobs:
245245

246246
- name: Run tests
247247
working-directory: ${{env.BUILD_DIR}}
248-
run: ctest -C ${{matrix.build_type}} --output-on-failure
248+
run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test
249249

250250
macos-build:
251251
name: MacOS

‎.github/workflows/benchmarks.yml

Lines changed: 58 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -7,125 +7,89 @@ permissions:
77
contents: read
88

99
jobs:
10-
benchmarks-ubuntu:
11-
name: Ubuntu
10+
benchmarks:
11+
name: Benchmarks
12+
env:
13+
BUILD_DIR : "${{github.workspace}}/build/"
14+
HWLOC_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows"
1215
strategy:
1316
matrix:
14-
os: ['ubuntu-22.04']
15-
build_type: [Release]
16-
compiler: [{c: gcc, cxx: g++}]
17-
shared_library: ['ON', 'OFF']
17+
include:
18+
- os: ubuntu-latest # shared library on Linux
19+
shared_library: 'ON'
20+
pool_scalable: 'ON'
21+
pool_jemalloc: 'ON'
22+
bench_mt: 'ON'
23+
24+
- os: ubuntu-latest # static library on Linux
25+
shared_library: 'OFF'
26+
pool_scalable: 'ON'
27+
pool_jemalloc: 'ON'
28+
bench_mt: 'ON'
29+
30+
- os: windows-latest # shared library on Windows
31+
shared_library: 'ON'
32+
pool_scalable: 'OFF'
33+
pool_jemalloc: 'OFF'
34+
bench_mt: 'OFF' # TODO: enable multithread_bench on Windows
35+
36+
- os: windows-latest # static library on Windows
37+
shared_library: 'OFF'
38+
pool_scalable: 'OFF'
39+
bench_mt: 'OFF' # TODO: enable multithread_bench on Windows
40+
1841
runs-on: ${{matrix.os}}
1942

2043
steps:
2144
- name: Checkout
2245
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
2346

2447
- name: Install apt packages
48+
if: matrix.os == 'ubuntu-latest'
2549
run: |
2650
sudo apt-get update
2751
sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
2852
53+
- name: Initialize vcpkg
54+
if: matrix.os == 'windows-latest'
55+
uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
56+
with:
57+
vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
58+
vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
59+
vcpkgJsonGlob: '**/vcpkg.json'
60+
61+
- name: Install dependencies
62+
if: matrix.os == 'windows-latest'
63+
run: vcpkg install
64+
shell: pwsh # Specifies PowerShell as the shell for running the script.
65+
2966
- name: Configure build
3067
run: >
3168
cmake
32-
-B ${{github.workspace}}/build
33-
-DCMAKE_BUILD_TYPE=${{matrix.build_type}}
34-
-DCMAKE_C_COMPILER=${{matrix.compiler.c}}
35-
-DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
69+
-B ${{env.BUILD_DIR}}
70+
-DCMAKE_BUILD_TYPE=Release
71+
-DCMAKE_PREFIX_PATH="${{env.HWLOC_PATH}}"
3672
-DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
3773
-DUMF_BUILD_BENCHMARKS=ON
38-
-DUMF_BUILD_BENCHMARKS_MT=ON
74+
-DUMF_BUILD_BENCHMARKS_MT=${{matrix.bench_mt}}
3975
-DUMF_BUILD_TESTS=OFF
4076
-DUMF_FORMAT_CODE_STYLE=OFF
4177
-DUMF_DEVELOPER_MODE=OFF
4278
-DUMF_BUILD_OS_MEMORY_PROVIDER=ON
43-
-DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
4479
-DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
45-
-DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
80+
-DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{matrix.pool_jemalloc}}
81+
-DUMF_BUILD_LIBUMF_POOL_SCALABLE=${{matrix.pool_scalable}}
4682
-DUMF_ENABLE_POOL_TRACKING=OFF
4783
-DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
4884
49-
- name: Build UMF
50-
run: cmake --build ${{github.workspace}}/build -j $(nproc)
85+
- name: Build UMF on Linux
86+
if: matrix.os == 'ubuntu-latest'
87+
run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
5188

89+
- name: Build UMF on Windows
90+
if: matrix.os == 'windows-latest'
91+
run: cmake --build ${{env.BUILD_DIR}} --config Release -j $Env:NUMBER_OF_PROCESSORS
92+
5293
- name: Run benchmarks
53-
working-directory: ${{github.workspace}}/build
54-
# The benchmark of TBB pool manager fails very often with the
55-
# "confidence interval exceeds maximum permitted 2.5%" error.
56-
# Do not treat that as a failure in CI.
57-
run: |
58-
export LOG=/tmp/ubench.log
59-
if ! ./benchmark/ubench 2>/dev/null > $LOG; then \
60-
cat $LOG; \
61-
if ! grep -q -e "exceeds maximum permitted 2.5" $LOG; then \
62-
echo "[ FAILED ] The CI benchmark job FAILED."; \
63-
exit 1; \
64-
fi; \
65-
fi
66-
cat $LOG
67-
echo "[ PASSED ] The CI benchmark job PASSED."
68-
69-
- name: Run MT benchmarks
70-
working-directory: ${{github.workspace}}/build
71-
run: ./benchmark/multithread_bench
72-
73-
benchmarks-windows:
74-
name: Windows
75-
env:
76-
BUILD_DIR : "${{github.workspace}}/build/"
77-
HWLOC_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows"
78-
strategy:
79-
fail-fast: false
80-
matrix:
81-
build_type: [Release]
82-
compiler: [{c: cl, cxx: cl}]
83-
shared_library: ['ON', 'OFF']
84-
runs-on: 'windows-2022'
85-
86-
steps:
87-
- name: Checkout
88-
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
89-
90-
- name: Initialize vcpkg
91-
uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
92-
with:
93-
vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
94-
vcpkgDirectory: ${{github.workspace}}/build/vcpkg
95-
vcpkgJsonGlob: '**/vcpkg.json'
96-
97-
- name: Install dependencies
98-
run: vcpkg install
99-
shell: pwsh # Specifies PowerShell as the shell for running the script.
100-
101-
- name: Configure build
102-
run: >
103-
cmake
104-
-B ${{env.BUILD_DIR}}
105-
-DCMAKE_C_COMPILER=${{matrix.compiler.c}}
106-
-DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
107-
-DCMAKE_PREFIX_PATH=${{env.HWLOC_PATH}}
108-
-DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
109-
-DUMF_BUILD_BENCHMARKS=ON
110-
-DUMF_BUILD_BENCHMARKS_MT=OFF
111-
-DUMF_BUILD_TESTS=OFF
112-
-DUMF_FORMAT_CODE_STYLE=OFF
113-
-DUMF_DEVELOPER_MODE=OFF
114-
-DUMF_BUILD_OS_MEMORY_PROVIDER=ON
115-
-DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
116-
-DUMF_BUILD_LIBUMF_POOL_JEMALLOC=OFF
117-
-DUMF_BUILD_LIBUMF_POOL_SCALABLE=OFF
118-
-DUMF_ENABLE_POOL_TRACKING=OFF
119-
120-
- name: Build UMF
121-
run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS
122-
123-
- name: Run benchmarks
124-
working-directory: ${{env.BUILD_DIR}}
125-
shell: pwsh # Specifies PowerShell as the shell for running the script.
126-
run: |
127-
# add path to umf.dll
128-
$env:PATH += ";${{env.BUILD_DIR}}\bin\${{matrix.build_type}}"
129-
# add path to hwloc-15.dll
130-
$env:PATH += ";${{env.HWLOC_PATH}}\bin"
131-
.\benchmark\${{matrix.build_type}}\ubench.exe
94+
working-directory: ${{env.BUILD_DIR}}
95+
run: ctest -V --test-dir benchmark -C Release

‎benchmark/CMakeLists.txt

Lines changed: 60 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -2,84 +2,84 @@
22
# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
33
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
44

5-
include(FindThreads)
6-
75
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
86
message(WARNING "The benchmarks SHOULD NOT be run in the Debug build type!")
97
endif()
108

11-
if(UMF_BUILD_LIBUMF_POOL_DISJOINT)
12-
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool)
13-
endif()
14-
15-
if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
16-
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool)
17-
endif()
18-
19-
if(UMF_BUILD_LIBUMF_POOL_SCALABLE)
20-
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} scalable_pool)
9+
if (UMF_BUILD_BENCHMARKS_MT)
10+
include(FindThreads)
2111
endif()
2212

23-
add_executable(ubench ubench.c)
13+
function(add_umf_benchmark)
14+
# NAME - a name of the benchmark
15+
# SRCS - source files
16+
# LIBS - libraries to be linked with
17+
set(oneValueArgs NAME)
18+
set(multiValueArgs SRCS LIBS)
19+
cmake_parse_arguments(ARG "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
2420

25-
if(LINUX)
26-
set(LIBS_LINUX m)
27-
endif()
21+
set(BENCH_NAME umf-bench-${ARG_NAME})
2822

29-
add_dependencies(ubench
30-
umf
31-
${LIBS_OPTIONAL})
23+
set(BENCH_LIBS umf ${ARG_LIBS})
3224

33-
target_include_directories(ubench PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include/ ${UMF_CMAKE_SOURCE_DIR}/src/utils/)
25+
add_umf_executable(NAME ${BENCH_NAME} SRCS ${ARG_SRCS} LIBS ${BENCH_LIBS})
3426

35-
target_link_libraries(ubench
36-
umf
37-
${LIBS_OPTIONAL}
38-
${CMAKE_THREAD_LIBS_INIT}
39-
${LIBS_LINUX})
27+
target_include_directories(${BENCH_NAME} PRIVATE
28+
${UMF_CMAKE_SOURCE_DIR}/include
29+
${UMF_CMAKE_SOURCE_DIR}/src/utils)
4030

41-
if (UMF_BUILD_BENCHMARKS_MT)
42-
add_executable(multithread_bench multithread.cpp)
43-
target_link_libraries(multithread_bench
44-
umf
45-
${LIBS_OPTIONAL}
46-
pthread
47-
${LIBS_LINUX})
48-
target_include_directories(multithread_bench PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include/)
49-
endif()
31+
add_test(NAME ${BENCH_NAME}
32+
COMMAND ${BENCH_NAME}
33+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
5034

51-
if (UMF_BUILD_OS_MEMORY_PROVIDER)
52-
target_compile_definitions(ubench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
35+
# Benchmark passes if it prints "PASSED" in the output,
36+
# because ubench of scalable pool fails if the confidence interval
37+
# exceeds maximum permitted 2.5%.
38+
set_tests_properties(${BENCH_NAME} PROPERTIES
39+
LABELS "benchmark"
40+
PASS_REGULAR_EXPRESSION "PASSED")
5341

54-
if (UMF_BUILD_BENCHMARKS_MT)
55-
target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
42+
if(WINDOWS)
43+
# append PATH to DLLs
44+
set(DLL_PATH_LIST "PATH=path_list_append:../bin/$<CONFIG>")
45+
if(LIBHWLOC_DLL_DIRS)
46+
set(DLL_PATH_LIST "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}")
5647
endif()
48+
set_property(TEST ${BENCH_NAME} PROPERTY ENVIRONMENT_MODIFICATION "${DLL_PATH_LIST}")
49+
endif()
50+
51+
if (UMF_BUILD_OS_MEMORY_PROVIDER)
52+
target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
53+
endif()
54+
if (UMF_BUILD_LIBUMF_POOL_DISJOINT)
55+
target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
56+
endif()
57+
if (UMF_BUILD_LIBUMF_POOL_JEMALLOC)
58+
target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
59+
endif()
60+
if (UMF_BUILD_LIBUMF_POOL_SCALABLE)
61+
target_compile_definitions(${BENCH_NAME} PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
62+
endif()
63+
endfunction()
64+
65+
# optional libraries
66+
if(UMF_BUILD_LIBUMF_POOL_DISJOINT)
67+
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool)
5768
endif()
58-
59-
if (UMF_BUILD_OS_MEMORY_PROVIDER)
60-
target_compile_definitions(ubench PRIVATE UMF_BUILD_OS_MEMORY_PROVIDER=1)
69+
if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
70+
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool)
6171
endif()
62-
63-
if (UMF_BUILD_LIBUMF_POOL_DISJOINT)
64-
target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
65-
66-
if (UMF_BUILD_BENCHMARKS_MT)
67-
target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_DISJOINT=1)
68-
endif()
72+
if(UMF_BUILD_LIBUMF_POOL_SCALABLE)
73+
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} scalable_pool)
6974
endif()
70-
71-
if (UMF_BUILD_LIBUMF_POOL_JEMALLOC)
72-
target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
73-
74-
if (UMF_BUILD_BENCHMARKS_MT)
75-
target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_JEMALLOC=1)
76-
endif()
75+
if(LINUX)
76+
set(LIBS_OPTIONAL ${LIBS_OPTIONAL} m)
7777
endif()
7878

79-
if (UMF_BUILD_LIBUMF_POOL_SCALABLE)
80-
target_compile_definitions(ubench PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
79+
# BENCHMARKS
8180

82-
if (UMF_BUILD_BENCHMARKS_MT)
83-
target_compile_definitions(multithread_bench PRIVATE UMF_BUILD_LIBUMF_POOL_SCALABLE=1)
84-
endif()
81+
add_umf_benchmark(NAME ubench SRCS ubench.c LIBS ${LIBS_OPTIONAL})
82+
83+
if (UMF_BUILD_BENCHMARKS_MT)
84+
add_umf_benchmark(NAME multithreaded SRCS multithread.cpp LIBS ${LIBS_OPTIONAL} pthread)
8585
endif()

0 commit comments

Comments
 (0)