Skip to content

Commit c80d2b2

Browse files
bratpiorkabratpiorka
authored
Merge pull request #1081 from lplewa/benchmark_stability
Improve benchmark stability.
2 parents 7b8c459 + 024b591 commit c80d2b2

File tree

3 files changed

+256
-264
lines changed

3 files changed

+256
-264
lines changed

‎benchmark/benchmark.cpp

Lines changed: 36 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -30,129 +30,60 @@
3030
// The exact meaning of each argument depends on the benchmark, allocator, and size components used.
3131
// Refer to the 'argsName()' function in each component to find detailed descriptions of these arguments.
3232

33-
static void default_alloc_fix_size(benchmark::internal::Benchmark *benchmark) {
34-
benchmark->Args({10000, 0, 4096});
35-
benchmark->Args({10000, 100000, 4096});
33+
static void multithreaded(benchmark::internal::Benchmark *benchmark) {
3634
benchmark->Threads(4);
3735
benchmark->Threads(1);
3836
}
3937

40-
static void
41-
default_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) {
42-
benchmark->Args({10000, 0, 8, 64 * 1024, 8});
43-
benchmark->Threads(4);
44-
benchmark->Threads(1);
45-
}
46-
47-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_fix, fixed_alloc_size,
48-
glibc_malloc);
49-
50-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_fix)
51-
->Apply(&default_alloc_fix_size);
52-
53-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, glibc_uniform,
54-
uniform_alloc_size, glibc_malloc);
55-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, glibc_uniform)
56-
->Apply(&default_alloc_uniform_size);
57-
58-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, os_provider, fixed_alloc_size,
59-
provider_allocator<os_provider>);
60-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, os_provider)
61-
->Apply(&default_alloc_fix_size);
62-
63-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, proxy_pool, fixed_alloc_size,
64-
pool_allocator<proxy_pool<os_provider>>);
65-
66-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, proxy_pool)
67-
->Apply(&default_alloc_fix_size);
68-
69-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_fix,
70-
fixed_alloc_size,
71-
pool_allocator<disjoint_pool<os_provider>>);
72-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_fix)
73-
->Apply(&default_alloc_fix_size);
74-
75-
// TODO: debug why this crashes
76-
/*UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, disjoint_pool_uniform,
77-
uniform_alloc_size,
78-
pool_allocator<disjoint_pool<os_provider>>);
79-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, disjoint_pool_uniform)
80-
->Apply(&default_alloc_uniform_size);
81-
*/
82-
83-
#ifdef UMF_POOL_JEMALLOC_ENABLED
84-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_fix,
85-
fixed_alloc_size,
86-
pool_allocator<jemalloc_pool<os_provider>>);
87-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_fix)
88-
->Apply(&default_alloc_fix_size);
89-
90-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, jemalloc_pool_uniform,
91-
uniform_alloc_size,
92-
pool_allocator<jemalloc_pool<os_provider>>);
93-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, jemalloc_pool_uniform)
94-
->Apply(&default_alloc_uniform_size);
95-
96-
#endif
97-
#ifdef UMF_POOL_SCALABLE_ENABLED
98-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_fix,
99-
fixed_alloc_size,
100-
pool_allocator<scalable_pool<os_provider>>);
101-
102-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_fix)
103-
->Apply(&default_alloc_fix_size);
104-
105-
UMF_BENCHMARK_TEMPLATE_DEFINE(alloc_benchmark, scalable_pool_uniform,
106-
uniform_alloc_size,
107-
pool_allocator<scalable_pool<os_provider>>);
108-
109-
UMF_BENCHMARK_REGISTER_F(alloc_benchmark, scalable_pool_uniform)
110-
->Apply(&default_alloc_uniform_size);
111-
#endif
112-
// Multiple allocs/free
11338
static void
11439
default_multiple_alloc_fix_size(benchmark::internal::Benchmark *benchmark) {
115-
benchmark->Args({10000, 4096});
116-
benchmark->Threads(4);
117-
benchmark->Threads(1);
40+
benchmark->Args({10000, 1, 4096});
41+
benchmark->Iterations(500000);
11842
}
11943

12044
static void
12145
default_multiple_alloc_uniform_size(benchmark::internal::Benchmark *benchmark) {
122-
benchmark->Args({10000, 8, 64 * 1024, 8});
123-
benchmark->Threads(4);
124-
benchmark->Threads(1);
46+
benchmark->Args({10000, 1, 8, 4096, 8});
47+
benchmark->Args({10000, 1, 8, 128, 8});
48+
benchmark->Iterations(500000);
12549
}
12650

12751
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_fix,
12852
fixed_alloc_size, glibc_malloc);
12953

13054
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_fix)
131-
->Apply(&default_multiple_alloc_fix_size);
55+
->Apply(&default_multiple_alloc_fix_size)
56+
->Apply(&multithreaded);
13257

13358
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, glibc_uniform,
13459
uniform_alloc_size, glibc_malloc);
13560
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, glibc_uniform)
136-
->Apply(&default_multiple_alloc_uniform_size);
61+
->Apply(&default_multiple_alloc_uniform_size)
62+
->Apply(&multithreaded);
13763

13864
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, proxy_pool,
13965
fixed_alloc_size,
14066
pool_allocator<proxy_pool<os_provider>>);
14167

14268
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, proxy_pool)
143-
->Apply(&default_multiple_alloc_fix_size);
69+
->Apply(&default_multiple_alloc_fix_size)
70+
// reduce iterations, as this benchmark is slower than others
71+
->Iterations(50000);
14472

14573
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, os_provider,
14674
fixed_alloc_size,
14775
provider_allocator<os_provider>);
14876
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, os_provider)
149-
->Apply(&default_multiple_alloc_fix_size);
77+
->Apply(&default_multiple_alloc_fix_size)
78+
// reduce iterations, as this benchmark is slower than others
79+
->Iterations(50000);
15080

15181
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, disjoint_pool_fix,
15282
fixed_alloc_size,
15383
pool_allocator<disjoint_pool<os_provider>>);
15484
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_fix)
155-
->Apply(&default_multiple_alloc_fix_size);
85+
->Apply(&default_multiple_alloc_fix_size)
86+
->Apply(&multithreaded);
15687

15788
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark,
15889
disjoint_pool_uniform, uniform_alloc_size,
@@ -165,13 +96,15 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, jemalloc_pool_fix,
16596
fixed_alloc_size,
16697
pool_allocator<jemalloc_pool<os_provider>>);
16798
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_fix)
168-
->Apply(&default_multiple_alloc_fix_size);
99+
->Apply(&default_multiple_alloc_fix_size)
100+
->Apply(&multithreaded);
169101

170102
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark,
171103
jemalloc_pool_uniform, uniform_alloc_size,
172104
pool_allocator<jemalloc_pool<os_provider>>);
173105
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, jemalloc_pool_uniform)
174-
->Apply(&default_multiple_alloc_uniform_size);
106+
->Apply(&default_multiple_alloc_uniform_size)
107+
->Apply(&multithreaded);
175108

176109
#endif
177110

@@ -181,14 +114,25 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark, scalable_pool_fix,
181114
pool_allocator<scalable_pool<os_provider>>);
182115

183116
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_fix)
184-
->Apply(&default_multiple_alloc_fix_size);
117+
->Apply(&default_multiple_alloc_fix_size)
118+
->Apply(&multithreaded);
185119

186120
UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark,
187121
scalable_pool_uniform, uniform_alloc_size,
188122
pool_allocator<scalable_pool<os_provider>>);
189123

190124
UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, scalable_pool_uniform)
191-
->Apply(&default_multiple_alloc_uniform_size);
125+
->Apply(&default_multiple_alloc_uniform_size)
126+
->Apply(&multithreaded);
192127

193128
#endif
194-
BENCHMARK_MAIN();
129+
130+
//BENCHMARK_MAIN();
131+
int main(int argc, char **argv) {
132+
if (initAffinityMask()) {
133+
return -1;
134+
}
135+
benchmark::Initialize(&argc, argv);
136+
benchmark::RunSpecifiedBenchmarks();
137+
benchmark::Shutdown();
138+
}

0 commit comments

Comments
 (0)