Skip to content

Commit 49011aa

Browse files
winner245winner245
committed
Optimize ranges::{for_each, for_each_n} for segmented iterators
1 parent 9ee950b commit 49011aa

File tree

7 files changed

+267
-53
lines changed

7 files changed

+267
-53
lines changed

‎libcxx/include/__algorithm/for_each_n.h

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@
1010
#ifndef _LIBCPP___ALGORITHM_FOR_EACH_N_H
1111
#define _LIBCPP___ALGORITHM_FOR_EACH_N_H
1212

13+
#include <__algorithm/for_each.h>
1314
#include <__config>
15+
#include <__iterator/iterator_traits.h>
16+
#include <__iterator/segmented_iterator.h>
17+
#include <__type_traits/enable_if.h>
1418
#include <__utility/convert_to_integral.h>
1519

1620
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -21,7 +25,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
2125

2226
#if _LIBCPP_STD_VER >= 17
2327

24-
template <class _InputIterator, class _Size, class _Function>
28+
template <class _InputIterator,
29+
class _Size,
30+
class _Function,
31+
__enable_if_t<!__is_segmented_iterator<_InputIterator>::value ||
32+
(__has_input_iterator_category<_InputIterator>::value &&
33+
!__has_random_access_iterator_category<_InputIterator>::value),
34+
int> = 0>
2535
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
2636
for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
2737
typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
@@ -34,6 +44,19 @@ for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
3444
return __first;
3545
}
3646

47+
template <class _InputIterator,
48+
class _Size,
49+
class _Function,
50+
__enable_if_t<__is_segmented_iterator<_InputIterator>::value &&
51+
__has_random_access_iterator_category<_InputIterator>::value,
52+
int> = 0>
53+
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
54+
for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
55+
_InputIterator __last = __first + __orig_n;
56+
std::for_each(__first, __last, __f);
57+
return __last;
58+
}
59+
3760
#endif
3861

3962
_LIBCPP_END_NAMESPACE_STD

‎libcxx/include/__algorithm/ranges_for_each.h

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
1010
#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
1111

12+
#include <__algorithm/for_each.h>
1213
#include <__algorithm/in_fun_result.h>
1314
#include <__config>
1415
#include <__functional/identity.h>
@@ -41,9 +42,16 @@ struct __for_each {
4142
template <class _Iter, class _Sent, class _Proj, class _Func>
4243
_LIBCPP_HIDE_FROM_ABI constexpr static for_each_result<_Iter, _Func>
4344
__for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
44-
for (; __first != __last; ++__first)
45-
std::invoke(__func, std::invoke(__proj, *__first));
46-
return {std::move(__first), std::move(__func)};
45+
if constexpr (random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
46+
auto __n = __last - __first;
47+
auto __end = __first + __n;
48+
std::for_each(__first, __end, [&](auto&& __val) { std::invoke(__func, std::invoke(__proj, __val)); });
49+
return {std::move(__end), std::move(__func)};
50+
} else {
51+
for (; __first != __last; ++__first)
52+
std::invoke(__func, std::invoke(__proj, *__first));
53+
return {std::move(__first), std::move(__func)};
54+
}
4755
}
4856

4957
public:

‎libcxx/include/__algorithm/ranges_for_each_n.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
1111

1212
#include <__algorithm/in_fun_result.h>
13+
#include <__algorithm/ranges_for_each.h>
1314
#include <__config>
1415
#include <__functional/identity.h>
1516
#include <__functional/invoke.h>
@@ -40,11 +41,16 @@ struct __for_each_n {
4041
template <input_iterator _Iter, class _Proj = identity, indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
4142
_LIBCPP_HIDE_FROM_ABI constexpr for_each_n_result<_Iter, _Func>
4243
operator()(_Iter __first, iter_difference_t<_Iter> __count, _Func __func, _Proj __proj = {}) const {
43-
while (__count-- > 0) {
44-
std::invoke(__func, std::invoke(__proj, *__first));
45-
++__first;
44+
if constexpr (random_access_iterator<_Iter>) {
45+
auto __last = __first + __count;
46+
return std::ranges::for_each(std::move(__first), std::move(__last), std::move(__func), std::move(__proj));
47+
} else {
48+
while (__count-- > 0) {
49+
std::invoke(__func, std::invoke(__proj, *__first));
50+
++__first;
51+
}
52+
return {std::move(__first), std::move(__func)};
4653
}
47-
return {std::move(__first), std::move(__func)};
4854
}
4955
};
5056

‎libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++03, c++11, c++14, c++17
10+
11+
#include <algorithm>
12+
#include <cstddef>
13+
#include <deque>
14+
#include <list>
15+
#include <string>
16+
#include <vector>
17+
18+
#include <benchmark/benchmark.h>
19+
20+
int main(int argc, char** argv) {
21+
auto std_for_each_n = [](auto first, auto n, auto f) { return std::for_each_n(first, n, f); };
22+
23+
// {std,ranges}::for_each_n
24+
{
25+
auto bm = []<class Container>(std::string name, auto for_each_n) {
26+
benchmark::RegisterBenchmark(
27+
name,
28+
[for_each_n](auto& st) {
29+
std::size_t const n = st.range(0);
30+
Container c(n, 1);
31+
auto first = c.begin();
32+
33+
for ([[maybe_unused]] auto _ : st) {
34+
benchmark::DoNotOptimize(c);
35+
auto result = for_each_n(first, n, [](int& x) { x = std::clamp(x, 10, 100); });
36+
benchmark::DoNotOptimize(result);
37+
}
38+
})
39+
->Arg(8)
40+
->Arg(32)
41+
->Arg(50) // non power-of-two
42+
->Arg(8192)
43+
->Arg(1 << 20);
44+
};
45+
bm.operator()<std::vector<int>>("std::for_each_n(vector<int>)", std_for_each_n);
46+
bm.operator()<std::deque<int>>("std::for_each_n(deque<int>)", std_for_each_n);
47+
bm.operator()<std::list<int>>("std::for_each_n(list<int>)", std_for_each_n);
48+
bm.operator()<std::vector<int>>("rng::for_each_n(vector<int>)", std::ranges::for_each_n);
49+
bm.operator()<std::deque<int>>("rng::for_each_n(deque<int>)", std::ranges::for_each_n);
50+
bm.operator()<std::list<int>>("rng::for_each_n(list<int>)", std::ranges::for_each_n);
51+
}
52+
53+
benchmark::Initialize(&argc, argv);
54+
benchmark::RunSpecifiedBenchmarks();
55+
benchmark::Shutdown();
56+
return 0;
57+
}

‎libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/for_each_n.pass.cpp

Lines changed: 81 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -13,69 +13,112 @@
1313
// constexpr InputIterator // constexpr after C++17
1414
// for_each_n(InputIterator first, Size n, Function f);
1515

16-
1716
#include <algorithm>
1817
#include <cassert>
18+
#include <deque>
1919
#include <functional>
20+
#include <iterator>
21+
#include <ranges>
2022

2123
#include "test_macros.h"
2224
#include "test_iterators.h"
2325

24-
#if TEST_STD_VER > 17
25-
TEST_CONSTEXPR bool test_constexpr() {
26-
int ia[] = {1, 3, 6, 7};
27-
int expected[] = {3, 5, 8, 9};
28-
const std::size_t N = 4;
26+
struct for_each_test {
27+
TEST_CONSTEXPR for_each_test(int c) : count(c) {}
28+
int count;
29+
TEST_CONSTEXPR_CXX14 void operator()(int& i) {
30+
++i;
31+
++count;
32+
}
33+
};
2934

30-
auto it = std::for_each_n(std::begin(ia), N, [](int &a) { a += 2; });
31-
return it == (std::begin(ia) + N)
32-
&& std::equal(std::begin(ia), std::end(ia), std::begin(expected))
33-
;
34-
}
35-
#endif
35+
struct deque_test {
36+
std::deque<int>* d_;
37+
int* i_;
38+
39+
deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
3640

37-
struct for_each_test
38-
{
39-
for_each_test(int c) : count(c) {}
40-
int count;
41-
void operator()(int& i) {++i; ++count;}
41+
void operator()(int& v) {
42+
assert(&(*d_)[*i_] == &v);
43+
++*i_;
44+
}
4245
};
4346

44-
int main(int, char**)
45-
{
47+
/*TEST_CONSTEXPR_CXX23*/
48+
void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX23 once std::deque is constexpr
49+
// check that segmented iterators work properly
50+
int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
51+
for (const int size : sizes) {
52+
std::deque<int> d(size);
53+
int index = 0;
54+
55+
std::for_each_n(d.begin(), d.size(), deque_test(d, index));
56+
}
57+
}
58+
59+
TEST_CONSTEXPR_CXX20 bool test() {
60+
{
4661
typedef cpp17_input_iterator<int*> Iter;
47-
int ia[] = {0, 1, 2, 3, 4, 5};
48-
const unsigned s = sizeof(ia)/sizeof(ia[0]);
62+
int ia[] = {0, 1, 2, 3, 4, 5};
63+
const unsigned s = sizeof(ia) / sizeof(ia[0]);
4964

5065
{
51-
auto f = for_each_test(0);
52-
Iter it = std::for_each_n(Iter(ia), 0, std::ref(f));
53-
assert(it == Iter(ia));
54-
assert(f.count == 0);
66+
auto f = for_each_test(0);
67+
Iter it = std::for_each_n(Iter(ia), 0, std::ref(f));
68+
assert(it == Iter(ia));
69+
assert(f.count == 0);
5570
}
5671

5772
{
58-
auto f = for_each_test(0);
59-
Iter it = std::for_each_n(Iter(ia), s, std::ref(f));
73+
auto f = for_each_test(0);
74+
Iter it = std::for_each_n(Iter(ia), s, std::ref(f));
6075

61-
assert(it == Iter(ia+s));
62-
assert(f.count == s);
63-
for (unsigned i = 0; i < s; ++i)
64-
assert(ia[i] == static_cast<int>(i+1));
76+
assert(it == Iter(ia + s));
77+
assert(f.count == s);
78+
for (unsigned i = 0; i < s; ++i)
79+
assert(ia[i] == static_cast<int>(i + 1));
6580
}
6681

6782
{
68-
auto f = for_each_test(0);
69-
Iter it = std::for_each_n(Iter(ia), 1, std::ref(f));
83+
auto f = for_each_test(0);
84+
Iter it = std::for_each_n(Iter(ia), 1, std::ref(f));
7085

71-
assert(it == Iter(ia+1));
72-
assert(f.count == 1);
73-
for (unsigned i = 0; i < 1; ++i)
74-
assert(ia[i] == static_cast<int>(i+2));
86+
assert(it == Iter(ia + 1));
87+
assert(f.count == 1);
88+
for (unsigned i = 0; i < 1; ++i)
89+
assert(ia[i] == static_cast<int>(i + 2));
7590
}
91+
}
92+
93+
#if TEST_STD_VER > 11
94+
{
95+
int ia[] = {1, 3, 6, 7};
96+
int expected[] = {3, 5, 8, 9};
97+
const std::size_t N = 4;
98+
99+
auto it = std::for_each_n(std::begin(ia), N, [](int& a) { a += 2; });
100+
assert(it == (std::begin(ia) + N) && std::equal(std::begin(ia), std::end(ia), std::begin(expected)));
101+
}
102+
#endif
103+
104+
if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_23_OR_RUNTIME_EVALUATED when std::deque is made constexpr
105+
test_segmented_deque_iterator();
106+
107+
#if TEST_STD_VER >= 20
108+
{ // Make sure that the segmented iterator optimization works during constant evaluation
109+
std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
110+
auto v = vec | std::views::join;
111+
std::for_each_n(v.begin(), std::ranges::distance(v), [i = 0](int& a) mutable { assert(a == i++); });
112+
}
113+
#endif
114+
115+
return true;
116+
}
76117

118+
int main(int, char**) {
119+
assert(test());
77120
#if TEST_STD_VER > 17
78-
static_assert(test_constexpr());
121+
static_assert(test());
79122
#endif
80123

81124
return 0;

0 commit comments

Comments
 (0)