|
9 | 9 | #include "src/string/memmove.h"
|
10 | 10 |
|
11 | 11 | #include "src/__support/common.h"
|
12 |
| -#include "src/string/memory_utils/op_aarch64.h" |
13 |
| -#include "src/string/memory_utils/op_builtin.h" |
14 |
| -#include "src/string/memory_utils/op_generic.h" |
15 |
| -#include "src/string/memory_utils/op_x86.h" |
| 12 | +#include "src/__support/integer_operations.h" |
| 13 | +#include "src/string/memory_utils/elements.h" |
16 | 14 | #include <stddef.h> // size_t, ptrdiff_t
|
17 | 15 |
|
18 |
| -#include <stdio.h> |
19 |
| - |
20 | 16 | namespace __llvm_libc {
|
21 | 17 |
|
22 |
| -[[maybe_unused]] static inline void |
23 |
| -inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { |
24 |
| - if ((count == 0) || (dst == src)) |
25 |
| - return; |
26 |
| - if (dst < src) { |
27 |
| -#pragma nounroll |
28 |
| - for (size_t offset = 0; offset < count; ++offset) |
29 |
| - builtin::Memcpy<1>::block(dst + offset, src + offset); |
30 |
| - } else { |
31 |
| -#pragma nounroll |
32 |
| - for (ptrdiff_t offset = count; offset >= 0; --offset) |
33 |
| - builtin::Memcpy<1>::block(dst + offset, src + offset); |
34 |
| - } |
35 |
| -} |
36 |
| - |
37 |
| -template <size_t MaxSize> |
38 |
| -[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, |
39 |
| - size_t count) { |
| 18 | +static inline void inline_memmove(char *dst, const char *src, size_t count) { |
| 19 | + using namespace __llvm_libc::scalar; |
40 | 20 | if (count == 0)
|
41 | 21 | return;
|
42 | 22 | if (count == 1)
|
43 |
| - return generic::Memmove<1, MaxSize>::block(dst, src); |
| 23 | + return move<_1>(dst, src); |
44 | 24 | if (count <= 4)
|
45 |
| - return generic::Memmove<2, MaxSize>::head_tail(dst, src, count); |
| 25 | + return move<HeadTail<_2>>(dst, src, count); |
46 | 26 | if (count <= 8)
|
47 |
| - return generic::Memmove<4, MaxSize>::head_tail(dst, src, count); |
| 27 | + return move<HeadTail<_4>>(dst, src, count); |
48 | 28 | if (count <= 16)
|
49 |
| - return generic::Memmove<8, MaxSize>::head_tail(dst, src, count); |
| 29 | + return move<HeadTail<_8>>(dst, src, count); |
50 | 30 | if (count <= 32)
|
51 |
| - return generic::Memmove<16, MaxSize>::head_tail(dst, src, count); |
| 31 | + return move<HeadTail<_16>>(dst, src, count); |
52 | 32 | if (count <= 64)
|
53 |
| - return generic::Memmove<32, MaxSize>::head_tail(dst, src, count); |
| 33 | + return move<HeadTail<_32>>(dst, src, count); |
54 | 34 | if (count <= 128)
|
55 |
| - return generic::Memmove<64, MaxSize>::head_tail(dst, src, count); |
56 |
| - if (dst < src) { |
57 |
| - generic::Memmove<32, MaxSize>::template align_forward<Arg::Src>(dst, src, |
58 |
| - count); |
59 |
| - return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src, |
60 |
| - count); |
61 |
| - } else { |
62 |
| - generic::Memmove<32, MaxSize>::template align_backward<Arg::Src>(dst, src, |
63 |
| - count); |
64 |
| - return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src, |
65 |
| - count); |
66 |
| - } |
67 |
| -} |
| 35 | + return move<HeadTail<_64>>(dst, src, count); |
68 | 36 |
|
69 |
| -static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { |
70 |
| -#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) |
71 |
| -#if defined(LLVM_LIBC_ARCH_X86) |
72 |
| - static constexpr size_t kMaxSize = x86::kAvx512F ? 64 |
73 |
| - : x86::kAvx ? 32 |
74 |
| - : x86::kSse2 ? 16 |
75 |
| - : 8; |
76 |
| -#elif defined(LLVM_LIBC_ARCH_AARCH64) |
77 |
| - static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8; |
78 |
| -#endif |
79 |
| - // return inline_memmove_generic<kMaxSize>(dst, src, count); |
80 |
| - if (count == 0) |
81 |
| - return; |
82 |
| - if (count == 1) |
83 |
| - return generic::Memmove<1, kMaxSize>::block(dst, src); |
84 |
| - if (count <= 4) |
85 |
| - return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count); |
86 |
| - if (count <= 8) |
87 |
| - return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count); |
88 |
| - if (count <= 16) |
89 |
| - return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count); |
90 |
| - if (count <= 32) |
91 |
| - return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count); |
92 |
| - if (count <= 64) |
93 |
| - return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count); |
94 |
| - if (count <= 128) |
95 |
| - return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count); |
96 |
| - if (dst < src) { |
97 |
| - generic::Memmove<32, kMaxSize>::align_forward<Arg::Src>(dst, src, count); |
98 |
| - return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src, |
99 |
| - count); |
100 |
| - } else { |
101 |
| - generic::Memmove<32, kMaxSize>::align_backward<Arg::Src>(dst, src, count); |
102 |
| - return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src, |
103 |
| - count); |
104 |
| - } |
105 |
| -#elif defined(LLVM_LIBC_ARCH_ARM) |
106 |
| - return inline_memmove_embedded_tiny(dst, src, count); |
107 |
| -#else |
108 |
| -#error "Unsupported platform" |
109 |
| -#endif |
| 37 | + using AlignedMoveLoop = Align<_16, Arg::Src>::Then<Loop<_64>>; |
| 38 | + if (dst < src) |
| 39 | + return move<AlignedMoveLoop>(dst, src, count); |
| 40 | + else if (dst > src) |
| 41 | + return move_backward<AlignedMoveLoop>(dst, src, count); |
110 | 42 | }
|
111 | 43 |
|
112 | 44 | LLVM_LIBC_FUNCTION(void *, memmove,
|
|
0 commit comments