Skip to content

Commit 48658ba

Browse files
SchrodingerZhuSchrodingerZhu
committed
[libc][c23] add memset_explicit
1 parent 8171f6d commit 48658ba

File tree

16 files changed

+280
-8
lines changed

16 files changed

+280
-8
lines changed

‎libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
set(ALL_CPU_FEATURES "")
77

88
if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
9-
set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX AVX2 AVX512F AVX512BW FMA)
9+
set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX AVX2 AVX512F AVX512BW FMA CLFLUSHOPT)
10+
set(CPU_FEATURES_DETECT_REQUIRES_RUN "CLFLUSHOPT")
1011
set(LIBC_COMPILE_OPTIONS_NATIVE -march=native)
1112
elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
13+
set(CPU_FEATURES_DETECT_REQUIRES_RUN "")
1214
set(LIBC_COMPILE_OPTIONS_NATIVE -mcpu=native)
1315
endif()
1416

@@ -53,12 +55,27 @@ else()
5355
# Try compile a C file to check if flag is supported.
5456
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
5557
foreach(feature IN LISTS ALL_CPU_FEATURES)
56-
try_compile(
57-
has_feature
58-
${CMAKE_CURRENT_BINARY_DIR}/cpu_features
59-
SOURCES ${LIBC_SOURCE_DIR}/cmake/modules/cpu_features/check_${feature}.cpp
60-
COMPILE_DEFINITIONS -I${LIBC_SOURCE_DIR} ${LIBC_COMPILE_OPTIONS_NATIVE}
61-
)
58+
if (${feature} IN_LIST CPU_FEATURES_DETECT_REQUIRES_RUN)
59+
try_run(
60+
return_code
61+
can_compile
62+
${CMAKE_CURRENT_BINARY_DIR}/cpu_features
63+
${LIBC_SOURCE_DIR}/cmake/modules/cpu_features/check_${feature}.cpp
64+
COMPILE_DEFINITIONS -I${LIBC_SOURCE_DIR} ${LIBC_COMPILE_OPTIONS_NATIVE}
65+
)
66+
if (can_compile AND return_code EQUAL 0)
67+
set(has_feature TRUE)
68+
else()
69+
set(has_feature FALSE)
70+
endif()
71+
else()
72+
try_compile(
73+
has_feature
74+
${CMAKE_CURRENT_BINARY_DIR}/cpu_features
75+
SOURCES ${LIBC_SOURCE_DIR}/cmake/modules/cpu_features/check_${feature}.cpp
76+
COMPILE_DEFINITIONS -I${LIBC_SOURCE_DIR} ${LIBC_COMPILE_OPTIONS_NATIVE}
77+
)
78+
endif()
6279
if(has_feature)
6380
list(APPEND AVAILABLE_CPU_FEATURES ${feature})
6481
endif()

‎libc/cmake/modules/cpu_features/check_CLFLUSHOPT.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
void test(char *ptr) { asm volatile("clflushopt %0" : "+m"(*ptr)::"memory"); }
2+
3+
int main(int argc, char **argv) {
4+
test(argv[0]);
5+
return 0;
6+
}

‎libc/config/linux/aarch64/entrypoints.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ set(TARGET_LIBC_ENTRYPOINTS
5151
libc.src.string.mempcpy
5252
libc.src.string.memrchr
5353
libc.src.string.memset
54+
libc.src.string.memset_explicit
5455
libc.src.string.rindex
5556
libc.src.string.stpcpy
5657
libc.src.string.stpncpy

‎libc/config/linux/x86_64/entrypoints.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ set(TARGET_LIBC_ENTRYPOINTS
5151
libc.src.string.mempcpy
5252
libc.src.string.memrchr
5353
libc.src.string.memset
54+
libc.src.string.memset_explicit
5455
libc.src.string.rindex
5556
libc.src.string.stpcpy
5657
libc.src.string.stpncpy

‎libc/spec/stdc.td

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,11 @@ def StdC : StandardSpec<"stdc"> {
234234
RetValSpec<VoidPtr>,
235235
[ArgSpec<VoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
236236
>,
237+
FunctionSpec<
238+
"memset_explicit",
239+
RetValSpec<VoidPtr>,
240+
[ArgSpec<VoidPtr>, ArgSpec<IntType>, ArgSpec<SizeTType>]
241+
>,
237242
FunctionSpec<
238243
"strcpy",
239244
RetValSpec<CharPtr>,

‎libc/src/string/CMakeLists.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,18 @@ add_entrypoint_object(
441441
.memory_utils.inline_memcpy
442442
)
443443

444+
add_entrypoint_object(
445+
memset_explicit
446+
SRCS
447+
memset_explicit.cpp
448+
HDRS
449+
memset_explicit.h
450+
DEPENDS
451+
.string_utils
452+
.memory_utils.inline_memset
453+
.memory_utils.flush_cache
454+
)
455+
444456
# Helper to define a function with multiple implementations
445457
# - Computes flags to satisfy required/rejected features and arch,
446458
# - Declares an entry point,

‎libc/src/string/memory_utils/CMakeLists.txt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ add_header_library(
77
aarch64/inline_memcpy.h
88
aarch64/inline_memmove.h
99
aarch64/inline_memset.h
10+
aarch64/flush_cacheline.h
1011
generic/aligned_access.h
1112
generic/byte_per_byte.h
1213
inline_bcmp.h
@@ -30,6 +31,7 @@ add_header_library(
3031
x86_64/inline_memcpy.h
3132
x86_64/inline_memmove.h
3233
x86_64/inline_memset.h
34+
x86_64/flush_cacheline.h
3335
DEPENDS
3436
libc.src.__support.common
3537
libc.src.__support.CPP.bit
@@ -97,3 +99,21 @@ add_header_library(
9799
HDRS
98100
inline_memmem.h
99101
)
102+
103+
if (CLFLUSHOPT IN_LIST LIBC_CPU_FEATURES)
104+
set(clflushopt_option "-DLIBC_TARGET_CPU_HAS_CLFLUSHOPT")
105+
message(STATUS "Using clflushopt for cacheline flushing")
106+
else()
107+
set(clflushopt_option "")
108+
endif()
109+
110+
add_header_library(
111+
flush_cache
112+
HDRS
113+
flush_cache.h
114+
COMPILE_OPTIONS
115+
${clflushopt_option}
116+
DEPENDS
117+
.memory_utils
118+
libc.src.__support.CPP.atomic
119+
)

‎libc/src/string/memory_utils/aarch64/flush_cacheline.h

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//===-- Flush Cacheline for AArch64 -----------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_FLUSH_CACHELINE_H
9+
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_FLUSH_CACHELINE_H
10+
11+
#include "src/__support/common.h"
12+
#include <stddef.h> // size_t
13+
namespace LIBC_NAMESPACE {
14+
15+
LIBC_INLINE size_t cacheline_size() {
16+
// Use the same way as in compiler-rt
17+
size_t ctr_el0;
18+
asm volatile("mrs %0, ctr_el0" : "=r"(ctr_el0));
19+
return 4 << ((ctr_el0 >> 16) & 15);
20+
}
21+
22+
LIBC_INLINE void flush_cacheline_async(volatile char *addr) {
23+
// flush to external memory and invalidate the cache line
24+
asm volatile("dc civac, %0" : : "r"(addr) : "memory");
25+
}
26+
27+
} // namespace LIBC_NAMESPACE
28+
29+
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_FLUSH_CACHELINE_H

‎libc/src/string/memory_utils/aarch64/inline_memset.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
namespace LIBC_NAMESPACE {
1919

20+
template <bool OPAQUE_VALUE = false>
2021
[[maybe_unused]] LIBC_INLINE static void
2122
inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
2223
static_assert(aarch64::kNeon, "aarch64 supports vector types");
@@ -45,7 +46,7 @@ inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
4546
generic::Memset<uint256_t>::tail(dst, value, count);
4647
return;
4748
}
48-
if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
49+
if (!OPAQUE_VALUE && count >= 448 && value == 0 && aarch64::neon::hasZva()) {
4950
generic::Memset<uint512_t>::block(dst, 0);
5051
align_to_next_boundary<64>(dst, count);
5152
return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);

‎libc/src/string/memory_utils/flush_cache.h

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
//===-- Dispatch cache flushing -------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_FLUSH_CACHE_H
10+
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_FLUSH_CACHE_H
11+
12+
#include "src/__support/CPP/atomic.h"
13+
#include "src/__support/common.h"
14+
#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_
15+
16+
#include <stddef.h> // size_t
17+
#include <stdint.h> // uintptr_t
18+
19+
#ifdef LIBC_TARGET_ARCH_IS_X86
20+
#include "src/string/memory_utils/x86_64/flush_cacheline.h"
21+
#define LIBC_HAS_FLUSH_CACHELINE_ASYNC 1
22+
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
23+
#include "src/string/memory_utils/aarch64/flush_cacheline.h"
24+
#define LIBC_HAS_FLUSH_CACHELINE_ASYNC 1
25+
#else
26+
#define LIBC_HAS_FLUSH_CACHELINE_ASYNC 0
27+
#endif
28+
29+
namespace LIBC_NAMESPACE {
30+
31+
LIBC_INLINE void flush_cache(volatile void *start, size_t size) {
32+
#if LIBC_HAS_FLUSH_CACHELINE_ASYNC
33+
size_t line_size = cacheline_size();
34+
uintptr_t addr = reinterpret_cast<uintptr_t>(start);
35+
uintptr_t offset = addr % line_size;
36+
// shift start to the left and align size to the right
37+
// we want to cover the whole range of memory that needs to be flushed
38+
size += offset;
39+
size += line_size - (size % line_size);
40+
addr -= offset;
41+
// flush cache line async may be reordered. We need to put barriers.
42+
cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
43+
for (size_t i = 0; i < size; i += line_size)
44+
flush_cacheline_async(reinterpret_cast<volatile char *>(addr + i));
45+
cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
46+
#else
47+
// we do not have specific instructions to flush the cache
48+
// fallback to use a full memory barrier instead.
49+
// Notice, however, memory fence might not flush the cache on many
50+
// architectures.
51+
cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
52+
#endif
53+
}
54+
55+
} // namespace LIBC_NAMESPACE
56+
#undef LIBC_HAS_FLUSH_CACHELINE_ASYNC
57+
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_FLUSH_CACHE_H

‎libc/src/string/memory_utils/inline_memset.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,16 @@
3636

3737
namespace LIBC_NAMESPACE {
3838

39+
template <bool OPAQUE_VALUE = false>
3940
LIBC_INLINE static void inline_memset(void *dst, uint8_t value, size_t count) {
41+
#if LIBC_TARGET_ARCH_IS_AARCH64
42+
// The AArch64 implementation has an additional template parameter. It
43+
// may uses dc zva to zero memory.
44+
LIBC_SRC_STRING_MEMORY_UTILS_MEMSET<OPAQUE_VALUE>(reinterpret_cast<Ptr>(dst),
45+
value, count);
46+
#else
4047
LIBC_SRC_STRING_MEMORY_UTILS_MEMSET(reinterpret_cast<Ptr>(dst), value, count);
48+
#endif
4149
}
4250

4351
} // namespace LIBC_NAMESPACE

‎libc/src/string/memory_utils/x86_64/flush_cacheline.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//===-- Flush Cacheline for x86_64 ------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_FLUSH_CACHELINE_H
9+
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_FLUSH_CACHELINE_H
10+
11+
#include "src/__support/common.h"
12+
#include <stddef.h> // size_t
13+
namespace LIBC_NAMESPACE {
14+
15+
LIBC_INLINE constexpr size_t cacheline_size() { return 64; }
16+
17+
LIBC_INLINE void flush_cacheline_async(volatile char *addr) {
18+
#if defined(LIBC_TARGET_CPU_HAS_CLFLUSHOPT)
19+
asm volatile("clflushopt %0" : "+m"(*addr)::"memory");
20+
#else
21+
__builtin_ia32_clflush(const_cast<const char *>(addr));
22+
#endif
23+
}
24+
25+
} // namespace LIBC_NAMESPACE
26+
27+
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_FLUSH_CACHELINE_H

‎libc/src/string/memset_explicit.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//===-- Implementation of memset_explicit ---------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "src/string/memset_explicit.h"
10+
#include "src/__support/common.h"
11+
#include "src/string/memory_utils/flush_cache.h"
12+
#include "src/string/memory_utils/inline_memset.h"
13+
14+
namespace LIBC_NAMESPACE {
15+
16+
LLVM_LIBC_FUNCTION(void *, memset_explicit,
17+
(void *dst, int value, size_t count)) {
18+
// Use the inline memset function to set the memory.
19+
inline_memset<true>(dst, static_cast<uint8_t>(value), count);
20+
21+
// Flush the cache line.
22+
flush_cache(dst, count);
23+
24+
return dst;
25+
}
26+
27+
} // namespace LIBC_NAMESPACE

‎libc/src/string/memset_explicit.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//===-- Implementation header for memset_explicit ---------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H
10+
#define LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H
11+
12+
#include <stddef.h> // size_t
13+
14+
namespace LIBC_NAMESPACE {
15+
16+
void *memset_explicit(void *ptr, int value, size_t count);
17+
18+
} // namespace LIBC_NAMESPACE
19+
20+
#endif // LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H

‎libc/test/src/string/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,16 @@ add_libc_test(
418418
libc.src.string.strxfrm
419419
)
420420

421+
add_libc_test(
422+
memset_explicit_test
423+
SUITE
424+
libc-string-tests
425+
SRCS
426+
memset_explicit_test.cpp
427+
DEPENDS
428+
libc.src.string.memset_explicit
429+
)
430+
421431
# Tests all implementations that can run on the target CPU.
422432
function(add_libc_multi_impl_test name)
423433
get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)

‎libc/test/src/string/memset_explicit_test.cpp

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
//===-- Unittests for memset_explicit -------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "memory_utils/memory_check_utils.h"
10+
#include "src/string/memset_explicit.h"
11+
#include "test/UnitTest/Test.h"
12+
13+
namespace LIBC_NAMESPACE {
14+
15+
// Apply the same tests as memset
16+
17+
static inline void Adaptor(cpp::span<char> p1, uint8_t value, size_t size) {
18+
LIBC_NAMESPACE::memset_explicit(p1.begin(), value, size);
19+
}
20+
21+
TEST(LlvmLibcmemsetExplicitTest, SizeSweep) {
22+
static constexpr size_t kMaxSize = 400;
23+
Buffer DstBuffer(kMaxSize);
24+
for (size_t size = 0; size < kMaxSize; ++size) {
25+
const char value = size % 10;
26+
auto dst = DstBuffer.span().subspan(0, size);
27+
ASSERT_TRUE((CheckMemset<Adaptor>(dst, value, size)));
28+
}
29+
}
30+
31+
} // namespace LIBC_NAMESPACE

0 commit comments

Comments
 (0)