Skip to content

Commit 2c5d9e3

Browse files
winner245winner245
committed
Optimize std::for_each_n for segmented iterators
1 parent 6de1537 commit 2c5d9e3

File tree

20 files changed

+350
-72
lines changed

20 files changed

+350
-72
lines changed

‎libcxx/docs/ReleaseNotes/21.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ Improvements and New Features
6363

6464
- The ``num_put::do_put`` integral overloads have been optimized, resulting in a performance improvement of up to 2.4x.
6565

66+
- The ``std::for_each_n`` algorithm has been optimized for segmented iterators, resulting in a performance improvement of
67+
up to 17.7x for ``std::deque<short>`` iterators, and up to 13.9x for ``std::join_view<vector<vector<short>>>`` iterators.
68+
6669
Deprecations and Removals
6770
-------------------------
6871

‎libcxx/include/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ set(files
2525
__algorithm/find_segment_if.h
2626
__algorithm/for_each.h
2727
__algorithm/for_each_n.h
28+
__algorithm/for_each_n_segment.h
2829
__algorithm/for_each_segment.h
2930
__algorithm/generate.h
3031
__algorithm/generate_n.h

‎libcxx/include/__algorithm/for_each.h

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,48 +9,43 @@
99

1010
#ifndef _LIBCPP___ALGORITHM_FOR_EACH_H
1111
#define _LIBCPP___ALGORITHM_FOR_EACH_H
12-
1312
#include <__algorithm/for_each_segment.h>
1413
#include <__config>
1514
#include <__iterator/segmented_iterator.h>
16-
#include <__ranges/movable_box.h>
17-
#include <__utility/in_place.h>
18-
#include <__utility/move.h>
15+
#include <__type_traits/enable_if.h>
1916

2017
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
2118
# pragma GCC system_header
2219
#endif
2320

24-
_LIBCPP_PUSH_MACROS
25-
#include <__undef_macros>
26-
2721
_LIBCPP_BEGIN_NAMESPACE_STD
2822

29-
template <class _InputIterator, class _Function>
30-
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
31-
for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
23+
template <class _InputIterator, class _Sent, class _Func>
24+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __for_each(_InputIterator __first, _Sent __last, _Func& __f) {
3225
for (; __first != __last; ++__first)
3326
__f(*__first);
34-
return __f;
3527
}
3628

37-
// __movable_box is available in C++20, but is actually a copyable-box, so optimization is only correct in C++23
38-
#if _LIBCPP_STD_VER >= 23
39-
template <class _SegmentedIterator, class _Function>
40-
requires __is_segmented_iterator<_SegmentedIterator>::value
41-
_LIBCPP_HIDE_FROM_ABI constexpr _Function
42-
for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function __func) {
43-
ranges::__movable_box<_Function> __wrapped_func(in_place, std::move(__func));
44-
std::__for_each_segment(__first, __last, [&](auto __lfirst, auto __llast) {
45-
__wrapped_func =
46-
ranges::__movable_box<_Function>(in_place, std::for_each(__lfirst, __llast, std::move(*__wrapped_func)));
29+
#ifndef _LIBCPP_CXX03_LANG
30+
template <class _SegmentedIterator,
31+
class _Function,
32+
__enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
33+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
34+
__for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function& __func) {
35+
using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
36+
std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
37+
std::__for_each(__lfirst, __llast, __func);
4738
});
48-
return std::move(*__wrapped_func);
4939
}
50-
#endif // _LIBCPP_STD_VER >= 23
40+
#endif // !_LIBCPP_CXX03_LANG
5141

52-
_LIBCPP_END_NAMESPACE_STD
42+
template <class _InputIterator, class _Function>
43+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
44+
for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
45+
std::__for_each(__first, __last, __f);
46+
return __f;
47+
}
5348

54-
_LIBCPP_POP_MACROS
49+
_LIBCPP_END_NAMESPACE_STD
5550

5651
#endif // _LIBCPP___ALGORITHM_FOR_EACH_H

‎libcxx/include/__algorithm/for_each_n.h

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,32 +10,88 @@
1010
#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_H
1111
#define _LIBCPP___ALGORITHM_FOR_EACH_N_H
1212

13+
#include <__algorithm/for_each.h>
14+
#include <__algorithm/for_each_n_segment.h>
1315
#include <__config>
16+
#include <__iterator/iterator_traits.h>
17+
#include <__iterator/segmented_iterator.h>
18+
#include <__type_traits/enable_if.h>
1419
#include <__utility/convert_to_integral.h>
20+
#include <__utility/move.h>
1521

1622
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
1723
# pragma GCC system_header
1824
#endif
1925

20-
_LIBCPP_BEGIN_NAMESPACE_STD
26+
_LIBCPP_PUSH_MACROS
27+
#include <__undef_macros>
2128

22-
#if _LIBCPP_STD_VER >= 17
29+
_LIBCPP_BEGIN_NAMESPACE_STD
2330

24-
template <class _InputIterator, class _Size, class _Function>
25-
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
26-
for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
31+
template <class _InputIterator,
32+
class _Size,
33+
class _Func,
34+
__enable_if_t<!__has_random_access_iterator_category<_InputIterator>::value &&
35+
(!__is_segmented_iterator<_InputIterator>::value
36+
// || !__has_random_access_iterator_category<
37+
// typename __segmented_iterator_traits<_InputIterator>::__local_iterator>::value
38+
), // TODO: __segmented_iterator_traits<_InputIterator> results in template instantiation
39+
// during SFINAE, which is a hard error to be fixed. Once fixed, we should uncomment.
40+
int> = 0>
41+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
42+
__for_each_n(_InputIterator __first, _Size __orig_n, _Func& __f) {
2743
typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
2844
_IntegralSize __n = __orig_n;
2945
while (__n > 0) {
3046
__f(*__first);
3147
++__first;
3248
--__n;
3349
}
34-
return __first;
50+
return std::move(__first);
3551
}
3652

37-
#endif
53+
template <class _RandIter,
54+
class _Size,
55+
class _Func,
56+
__enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
57+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandIter
58+
__for_each_n(_RandIter __first, _Size __orig_n, _Func& __f) {
59+
typename std::iterator_traits<_RandIter>::difference_type __n = __orig_n;
60+
auto __last = __first + __n;
61+
std::__for_each(__first, __last, __f);
62+
return std::move(__last);
63+
}
64+
65+
#ifndef _LIBCPP_CXX03_LANG
66+
template <class _SegmentedIterator,
67+
class _Size,
68+
class _Func,
69+
__enable_if_t<!__has_random_access_iterator_category<_SegmentedIterator>::value &&
70+
__is_segmented_iterator<_SegmentedIterator>::value &&
71+
__has_random_access_iterator_category<
72+
typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator>::value,
73+
int> = 0>
74+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _SegmentedIterator
75+
__for_each_n(_SegmentedIterator __first, _Size __orig_n, _Func& __f) {
76+
using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
77+
return std::__for_each_n_segment(__first, __orig_n, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
78+
std::__for_each(__lfirst, __llast, __f);
79+
});
80+
}
81+
#endif // !_LIBCPP_CXX03_LANG
82+
83+
#if _LIBCPP_STD_VER >= 17
84+
85+
template <class _InputIterator, class _Size, class _Function>
86+
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
87+
for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
88+
return std::__for_each_n(__first, __orig_n, __f);
89+
}
90+
91+
#endif // _LIBCPP_STD_VER >= 17
3892

3993
_LIBCPP_END_NAMESPACE_STD
4094

95+
_LIBCPP_POP_MACROS
96+
4197
#endif // _LIBCPP___ALGORITHM_FOR_EACH_N_H

‎libcxx/include/__algorithm/for_each_n_segment.h

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_SEGMENT_H
10+
#define _LIBCPP___ALGORITHM_FOR_EACH_N_SEGMENT_H
11+
12+
#include <__config>
13+
#include <__iterator/iterator_traits.h>
14+
#include <__iterator/segmented_iterator.h>
15+
16+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17+
# pragma GCC system_header
18+
#endif
19+
20+
_LIBCPP_BEGIN_NAMESPACE_STD
21+
22+
// __for_each_n_segment optimizes linear iteration over segmented iterators. It processes a segmented
23+
// input range [__first, __first + __n) by applying the functor __func to each element within the segment.
24+
// The return value of __func is ignored, and the function returns an iterator pointing to one past the
25+
// last processed element in the input range.
26+
27+
template <class _SegmentedIterator, class _Size, class _Functor>
28+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _SegmentedIterator
29+
__for_each_n_segment(_SegmentedIterator __first, _Size __orig_n, _Functor __func) {
30+
static_assert(__is_segmented_iterator<_SegmentedIterator>::value &&
31+
__has_random_access_iterator_category<
32+
typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator>::value,
33+
"__for_each_n_segment only works with segmented iterators with random-access local iterators");
34+
if (__orig_n <= 0)
35+
return __first;
36+
37+
using _Traits = __segmented_iterator_traits<_SegmentedIterator>;
38+
using __local_iter_t = typename _Traits::__local_iterator;
39+
using __difference_t = typename std::iterator_traits<__local_iter_t>::difference_type;
40+
__difference_t __n = __orig_n;
41+
auto __seg = _Traits::__segment(__first);
42+
auto __local_first = _Traits::__local(__first);
43+
__local_iter_t __local_last;
44+
45+
while (__n > 0) {
46+
__local_last = _Traits::__end(__seg);
47+
auto __seg_size = __local_last - __local_first;
48+
if (__n <= __seg_size) {
49+
__local_last = __local_first + __n;
50+
__func(__local_first, __local_last);
51+
break;
52+
}
53+
__func(__local_first, __local_last);
54+
__n -= __seg_size;
55+
__local_first = _Traits::__begin(++__seg);
56+
}
57+
58+
return _Traits::__compose(__seg, __local_last);
59+
}
60+
61+
_LIBCPP_END_NAMESPACE_STD
62+
63+
#endif // _LIBCPP___ALGORITHM_FOR_EACH_N_SEGMENT_H

‎libcxx/include/algorithm

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2061,6 +2061,7 @@ template <class BidirectionalIterator, class Compare>
20612061
# include <cstring>
20622062
# include <iterator>
20632063
# include <memory>
2064+
# include <optional>
20642065
# include <stdexcept>
20652066
# include <type_traits>
20662067
# include <utility>

‎libcxx/include/array

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,7 @@ _LIBCPP_POP_MACROS
566566
# include <cstdlib>
567567
# include <iterator>
568568
# include <new>
569+
# include <optional>
569570
# include <type_traits>
570571
# include <utility>
571572
# endif

‎libcxx/include/bitset

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -973,6 +973,7 @@ _LIBCPP_POP_MACROS
973973
# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
974974
# include <concepts>
975975
# include <cstdlib>
976+
# include <optional>
976977
# include <type_traits>
977978
# endif
978979
#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)

‎libcxx/include/codecvt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,7 @@ _LIBCPP_END_NAMESPACE_STD
596596
# include <limits>
597597
# include <mutex>
598598
# include <new>
599+
# include <optional>
599600
# include <stdexcept>
600601
# include <type_traits>
601602
# include <typeinfo>

‎libcxx/include/condition_variable

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ _LIBCPP_POP_MACROS
357357
# include <initializer_list>
358358
# include <iosfwd>
359359
# include <new>
360+
# include <optional>
360361
# include <stdexcept>
361362
# include <system_error>
362363
# include <type_traits>

‎libcxx/include/ios

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -887,6 +887,7 @@ _LIBCPP_POP_MACROS
887887
# include <limits>
888888
# include <mutex>
889889
# include <new>
890+
# include <optional>
890891
# include <stdexcept>
891892
# include <system_error>
892893
# include <type_traits>

‎libcxx/include/locale

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3692,6 +3692,7 @@ _LIBCPP_POP_MACROS
36923692
# include <cstdarg>
36933693
# include <iterator>
36943694
# include <mutex>
3695+
# include <optional>
36953696
# include <stdexcept>
36963697
# include <type_traits>
36973698
# include <typeinfo>

‎libcxx/include/module.modulemap

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,7 @@ module std [system] {
436436
module find_segment_if { header "__algorithm/find_segment_if.h" }
437437
module find { header "__algorithm/find.h" }
438438
module for_each_n { header "__algorithm/for_each_n.h" }
439+
module for_each_n_segment { header "__algorithm/for_each_n_segment.h" }
439440
module for_each_segment { header "__algorithm/for_each_segment.h" }
440441
module for_each { header "__algorithm/for_each.h" }
441442
module generate_n { header "__algorithm/generate_n.h" }

‎libcxx/include/streambuf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,7 @@ _LIBCPP_POP_MACROS
386386

387387
# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
388388
# include <cstdint>
389+
# include <optional>
389390
# endif
390391
#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
391392

‎libcxx/include/string

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4027,6 +4027,7 @@ _LIBCPP_POP_MACROS
40274027
# include <cstdlib>
40284028
# include <iterator>
40294029
# include <new>
4030+
# include <optional>
40304031
# include <type_traits>
40314032
# include <typeinfo>
40324033
# include <utility>

‎libcxx/include/string_view

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,7 @@ _LIBCPP_POP_MACROS
952952
# include <concepts>
953953
# include <cstdlib>
954954
# include <iterator>
955+
# include <optional>
955956
# include <type_traits>
956957
# endif
957958
#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)

‎libcxx/include/system_error

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ template <> struct hash<std::error_condition>;
168168
# include <cstdint>
169169
# include <cstring>
170170
# include <limits>
171+
# include <optional>
171172
# include <type_traits>
172173
# endif
173174
#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)

‎libcxx/include/vector

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,7 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
362362
# if _LIBCPP_HAS_LOCALIZATION
363363
# include <locale>
364364
# endif
365+
# include <optional>
365366
# include <string>
366367
# include <string_view>
367368
# include <tuple>

0 commit comments

Comments
 (0)