Skip to content

Commit 3085e42

Browse files
itrofimowmordante
itrofimow
authored and
mordante
committed
[libc++] Don't call key_eq in unordered_map/set rehashing routine
As of now containers key_eq might get called when rehashing happens, which is redundant for unique keys containers. Reviewed By: #libc, philnik, Mordante Differential Revision: https://reviews.llvm.org/D128021
1 parent 393e12b commit 3085e42

File tree

7 files changed

+143
-81
lines changed

7 files changed

+143
-81
lines changed

‎libcxx/benchmarks/ContainerBenchmarks.h

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,20 @@ static void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
135135
}
136136
}
137137

138+
template <class Container, class GenInputs>
139+
static void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
140+
auto in = gen(st.range(0));
141+
c.max_load_factor(3.0);
142+
c.insert(in.begin(), in.end());
143+
benchmark::DoNotOptimize(c);
144+
const auto bucket_count = c.bucket_count();
145+
while (st.KeepRunning()) {
146+
c.rehash(bucket_count + 1);
147+
c.rehash(bucket_count);
148+
benchmark::ClobberMemory();
149+
}
150+
}
151+
138152
} // end namespace ContainerBenchmarks
139153

140154
#endif // BENCHMARK_CONTAINER_BENCHMARKS_H

‎libcxx/benchmarks/unordered_set_operations.bench.cpp

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,27 @@ struct UInt64Hash2 {
104104
}
105105
};
106106

107+
// The sole purpose of this comparator is to be used in BM_Rehash, where
108+
// we need something slow enough to be easily noticable in benchmark results.
109+
// The default implementation of operator== for strings seems to be a little
110+
// too fast for that specific benchmark to reliably show a noticeable
111+
// improvement, but unoptimized bytewise comparison fits just right.
112+
// Early return is there just for convenience, since we only compare strings
113+
// of equal length in BM_Rehash.
114+
struct SlowStringEq {
115+
SlowStringEq() = default;
116+
inline TEST_ALWAYS_INLINE
117+
bool operator()(const std::string& lhs, const std::string& rhs) const {
118+
if (lhs.size() != rhs.size()) return false;
119+
120+
bool eq = true;
121+
for (size_t i = 0; i < lhs.size(); ++i) {
122+
eq &= lhs[i] == rhs[i];
123+
}
124+
return eq;
125+
}
126+
};
127+
107128
//----------------------------------------------------------------------------//
108129
// BM_Hash
109130
// ---------------------------------------------------------------------------//
@@ -266,6 +287,20 @@ BENCHMARK_CAPTURE(BM_FindRehash,
266287
std::unordered_set<std::string>{},
267288
getRandomStringInputs)->Arg(TestNumInputs);
268289

290+
//----------------------------------------------------------------------------//
291+
// BM_Rehash
292+
// ---------------------------------------------------------------------------//
293+
294+
BENCHMARK_CAPTURE(BM_Rehash,
295+
unordered_set_string_arg,
296+
std::unordered_set<std::string, std::hash<std::string>, SlowStringEq>{},
297+
getRandomStringInputs)->Arg(TestNumInputs);
298+
299+
BENCHMARK_CAPTURE(BM_Rehash,
300+
unordered_set_int_arg,
301+
std::unordered_set<int>{},
302+
getRandomIntegerInputs<int>)->Arg(TestNumInputs);
303+
269304
///////////////////////////////////////////////////////////////////////////////
270305
BENCHMARK_CAPTURE(BM_InsertDuplicate,
271306
unordered_set_int,

‎libcxx/include/__hash_table

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1146,9 +1146,16 @@ public:
11461146
#endif
11471147

11481148
void clear() _NOEXCEPT;
1149-
void rehash(size_type __n);
1150-
_LIBCPP_INLINE_VISIBILITY void reserve(size_type __n)
1151-
{rehash(static_cast<size_type>(ceil(__n / max_load_factor())));}
1149+
_LIBCPP_INLINE_VISIBILITY void __rehash_unique(size_type __n) { __rehash<true>(__n); }
1150+
_LIBCPP_INLINE_VISIBILITY void __rehash_multi(size_type __n) { __rehash<false>(__n); }
1151+
_LIBCPP_INLINE_VISIBILITY void __reserve_unique(size_type __n)
1152+
{
1153+
__rehash_unique(static_cast<size_type>(ceil(__n / max_load_factor())));
1154+
}
1155+
_LIBCPP_INLINE_VISIBILITY void __reserve_multi(size_type __n)
1156+
{
1157+
__rehash_multi(static_cast<size_type>(ceil(__n / max_load_factor())));
1158+
}
11521159

11531160
_LIBCPP_INLINE_VISIBILITY
11541161
size_type bucket_count() const _NOEXCEPT
@@ -1285,7 +1292,8 @@ public:
12851292
#endif // _LIBCPP_ENABLE_DEBUG_MODE
12861293

12871294
private:
1288-
void __rehash(size_type __n);
1295+
template <bool _UniqueKeys> void __rehash(size_type __n);
1296+
template <bool _UniqueKeys> void __do_rehash(size_type __n);
12891297

12901298
template <class ..._Args>
12911299
__node_holder __construct_node(_Args&& ...__args);
@@ -1790,7 +1798,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare(
17901798
}
17911799
if (size()+1 > __bc * max_load_factor() || __bc == 0)
17921800
{
1793-
rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
1801+
__rehash_unique(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
17941802
size_type(ceil(float(size() + 1) / max_load_factor()))));
17951803
}
17961804
return nullptr;
@@ -1862,7 +1870,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare(
18621870
size_type __bc = bucket_count();
18631871
if (size()+1 > __bc * max_load_factor() || __bc == 0)
18641872
{
1865-
rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
1873+
__rehash_multi(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
18661874
size_type(ceil(float(size() + 1) / max_load_factor()))));
18671875
__bc = bucket_count();
18681876
}
@@ -1956,7 +1964,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(
19561964
size_type __bc = bucket_count();
19571965
if (size()+1 > __bc * max_load_factor() || __bc == 0)
19581966
{
1959-
rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
1967+
__rehash_multi(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
19601968
size_type(ceil(float(size() + 1) / max_load_factor()))));
19611969
__bc = bucket_count();
19621970
}
@@ -2004,7 +2012,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const&
20042012
__node_holder __h = __construct_node_hash(__hash, _VSTD::forward<_Args>(__args)...);
20052013
if (size()+1 > __bc * max_load_factor() || __bc == 0)
20062014
{
2007-
rehash(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
2015+
__rehash_unique(_VSTD::max<size_type>(2 * __bc + !__is_hash_power2(__bc),
20082016
size_type(ceil(float(size() + 1) / max_load_factor()))));
20092017
__bc = bucket_count();
20102018
__chash = __constrain_hash(__hash, __bc);
@@ -2207,8 +2215,9 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi(
22072215
#endif // _LIBCPP_STD_VER > 14
22082216

22092217
template <class _Tp, class _Hash, class _Equal, class _Alloc>
2218+
template <bool _UniqueKeys>
22102219
void
2211-
__hash_table<_Tp, _Hash, _Equal, _Alloc>::rehash(size_type __n)
2220+
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n)
22122221
_LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
22132222
{
22142223
if (__n == 1)
@@ -2217,7 +2226,7 @@ _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
22172226
__n = __next_prime(__n);
22182227
size_type __bc = bucket_count();
22192228
if (__n > __bc)
2220-
__rehash(__n);
2229+
__do_rehash<_UniqueKeys>(__n);
22212230
else if (__n < __bc)
22222231
{
22232232
__n = _VSTD::max<size_type>
@@ -2227,13 +2236,14 @@ _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
22272236
__next_prime(size_t(ceil(float(size()) / max_load_factor())))
22282237
);
22292238
if (__n < __bc)
2230-
__rehash(__n);
2239+
__do_rehash<_UniqueKeys>(__n);
22312240
}
22322241
}
22332242

22342243
template <class _Tp, class _Hash, class _Equal, class _Alloc>
2244+
template <bool _UniqueKeys>
22352245
void
2236-
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __nbc)
2246+
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc)
22372247
{
22382248
std::__debug_db_invalidate_all(this);
22392249
__pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc();
@@ -2268,11 +2278,14 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __nbc)
22682278
else
22692279
{
22702280
__next_pointer __np = __cp;
2271-
for (; __np->__next_ != nullptr &&
2272-
key_eq()(__cp->__upcast()->__value_,
2273-
__np->__next_->__upcast()->__value_);
2274-
__np = __np->__next_)
2275-
;
2281+
if _LIBCPP_CONSTEXPR_AFTER_CXX14 (!_UniqueKeys)
2282+
{
2283+
for (; __np->__next_ != nullptr &&
2284+
key_eq()(__cp->__upcast()->__value_,
2285+
__np->__next_->__upcast()->__value_);
2286+
__np = __np->__next_)
2287+
;
2288+
}
22762289
__pp->__next_ = __np->__next_;
22772290
__np->__next_ = __bucket_list_[__chash]->__next_;
22782291
__bucket_list_[__chash]->__next_ = __cp;

‎libcxx/include/ext/hash_map

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -605,7 +605,7 @@ public:
605605
{return __table_.bucket_size(__n);}
606606

607607
_LIBCPP_INLINE_VISIBILITY
608-
void resize(size_type __n) {__table_.rehash(__n);}
608+
void resize(size_type __n) {__table_.__rehash_unique(__n);}
609609

610610
private:
611611
__node_holder __construct_node(const key_type& __k);
@@ -616,7 +616,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
616616
size_type __n, const hasher& __hf, const key_equal& __eql)
617617
: __table_(__hf, __eql)
618618
{
619-
__table_.rehash(__n);
619+
__table_.__rehash_unique(__n);
620620
}
621621

622622
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -625,7 +625,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
625625
const allocator_type& __a)
626626
: __table_(__hf, __eql, __a)
627627
{
628-
__table_.rehash(__n);
628+
__table_.__rehash_unique(__n);
629629
}
630630

631631
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -643,7 +643,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
643643
const hasher& __hf, const key_equal& __eql)
644644
: __table_(__hf, __eql)
645645
{
646-
__table_.rehash(__n);
646+
__table_.__rehash_unique(__n);
647647
insert(__first, __last);
648648
}
649649

@@ -654,7 +654,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
654654
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
655655
: __table_(__hf, __eql, __a)
656656
{
657-
__table_.rehash(__n);
657+
__table_.__rehash_unique(__n);
658658
insert(__first, __last);
659659
}
660660

@@ -663,7 +663,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
663663
const hash_map& __u)
664664
: __table_(__u.__table_)
665665
{
666-
__table_.rehash(__u.bucket_count());
666+
__table_.__rehash_unique(__u.bucket_count());
667667
insert(__u.begin(), __u.end());
668668
}
669669

@@ -874,15 +874,15 @@ public:
874874
{return __table_.bucket_size(__n);}
875875

876876
_LIBCPP_INLINE_VISIBILITY
877-
void resize(size_type __n) {__table_.rehash(__n);}
877+
void resize(size_type __n) {__table_.__rehash_multi(__n);}
878878
};
879879

880880
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
881881
hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
882882
size_type __n, const hasher& __hf, const key_equal& __eql)
883883
: __table_(__hf, __eql)
884884
{
885-
__table_.rehash(__n);
885+
__table_.__rehash_multi(__n);
886886
}
887887

888888
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -891,7 +891,7 @@ hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
891891
const allocator_type& __a)
892892
: __table_(__hf, __eql, __a)
893893
{
894-
__table_.rehash(__n);
894+
__table_.__rehash_multi(__n);
895895
}
896896

897897
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -909,7 +909,7 @@ hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
909909
const hasher& __hf, const key_equal& __eql)
910910
: __table_(__hf, __eql)
911911
{
912-
__table_.rehash(__n);
912+
__table_.__rehash_multi(__n);
913913
insert(__first, __last);
914914
}
915915

@@ -920,7 +920,7 @@ hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
920920
const hasher& __hf, const key_equal& __eql, const allocator_type& __a)
921921
: __table_(__hf, __eql, __a)
922922
{
923-
__table_.rehash(__n);
923+
__table_.__rehash_multi(__n);
924924
insert(__first, __last);
925925
}
926926

@@ -929,7 +929,7 @@ hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
929929
const hash_multimap& __u)
930930
: __table_(__u.__table_)
931931
{
932-
__table_.rehash(__u.bucket_count());
932+
__table_.__rehash_multi(__u.bucket_count());
933933
insert(__u.begin(), __u.end());
934934
}
935935

0 commit comments

Comments
 (0)