address CRs and improve inlining of fastpaths

Author: Yifan Zhu

libc/src/__support/OSUtil/linux/aarch64/vdso.h

@@ -10,23 +10,6 @@
 #include "src/__support/CPP/string_view.h"
 namespace LIBC_NAMESPACE {
 namespace vdso {
-// macro definitions
-#define LIBC_VDSO_HAS_RT_SIGRETURN
-#define LIBC_VDSO_HAS_GETTIMEOFDAY
-#define LIBC_VDSO_HAS_CLOCK_GETTIME
-#define LIBC_VDSO_HAS_CLOCK_GETRES
-
-// list of VDSO symbols
-// following the order in
-// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/kernel/vdso/vdso.lds.S
-enum class VDSOSym {
-  RTSigReturn,
-  GetTimeOfDay,
-  ClockGetTime,
-  ClockGetRes,
-  VDSOSymCount
-};
-
 // translate VDSOSym to symbol names
 LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
   switch (sym) {

libc/src/__support/OSUtil/linux/arm/vdso.h

@@ -10,23 +10,6 @@
 #include "src/__support/CPP/string_view.h"
 namespace LIBC_NAMESPACE {
 namespace vdso {
-// macro definitions
-// following the order in
-// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm/vdso/vdso.lds.S
-#define LIBC_VDSO_HAS_CLOCK_GETTIME
-#define LIBC_VDSO_HAS_GETTIMEOFDAY
-#define LIBC_VDSO_HAS_CLOCK_GETRES
-#define LIBC_VDSO_HAS_CLOCK_GETTIME64
-
-// list of VDSO symbols
-enum class VDSOSym {
-  ClockGetTime,
-  GetTimeOfDay,
-  ClockGetRes,
-  ClockGetTime64,
-  VDSOSymCount
-};
-
 // translate VDSOSym to symbol names
 LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
   switch (sym) {

libc/src/__support/OSUtil/linux/riscv/vdso.h

@@ -10,29 +10,6 @@
 #include "src/__support/CPP/string_view.h"
 namespace LIBC_NAMESPACE {
 namespace vdso {
-// macro definitions
-// following the order in
-// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/kernel/vdso/vdso.lds.S
-#define LIBC_VDSO_HAS_RT_SIGRETURN
-#define LIBC_VDSO_HAS_GETTIMEOFDAY
-#define LIBC_VDSO_HAS_CLOCK_GETTIME
-#define LIBC_VDSO_HAS_CLOCK_GETRES
-#define LIBC_VDSO_HAS_GETCPU
-#define LIBC_VDSO_HAS_FLUSH_ICACHE
-#define LIBC_VDSO_HAS_RISCV_HWPROBE
-
-// list of VDSO symbols
-enum class VDSOSym {
-  RTSigReturn,
-  GetTimeOfDay,
-  ClockGetTime,
-  ClockGetRes,
-  GetCpu,
-  FlushICache,
-  RiscvHwProbe,
-  VDSOSymCount
-};
-
 // translate VDSOSym to symbol names
 LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
   switch (sym) {

libc/src/__support/OSUtil/linux/vdso.cpp

@@ -34,24 +34,10 @@ unsigned long getauxval(unsigned long id);
 
 namespace vdso {
 
-namespace {
-
-// This class is an internal helper for symbol <-> index conversion and
-// improve the readability of the code.
-// We keep this class internal on purpose to avoid users from implicitly
-// converting between size_t and VDSOSym.
-class Symbol {
-  VDSOSym sym;
-
-public:
-  static constexpr size_t COUNT = static_cast<size_t>(VDSOSym::VDSOSymCount);
-  constexpr Symbol(VDSOSym sym) : sym(sym) {}
-  constexpr Symbol(size_t idx) : sym(static_cast<VDSOSym>(idx)) {}
-  constexpr cpp::string_view name() const { return symbol_name(sym); }
-  constexpr cpp::string_view version() const { return symbol_version(sym); }
-  constexpr operator size_t() const { return static_cast<size_t>(sym); }
-};
+Symbol::VDSOArray Symbol::global_cache{};
+CallOnceFlag Symbol::once_flag = callonce_impl::NOT_CALLED;
 
+namespace {
 // See https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symverdefs.html
 struct Verdaux {
   ElfW(Word) vda_name; /* Version or dependency names */
@@ -101,10 +87,6 @@ cpp::string_view find_version(Verdef *verdef, ElfW(Half) * versym,
   return "";
 }
 
-using VDSOArray = cpp::array<void *, Symbol::COUNT>;
-
-VDSOArray symbol_table;
-
 size_t shdr_get_symbol_count(ElfW(Shdr) * vdso_shdr, size_t e_shnum) {
   // iterate all sections until we locate the dynamic symbol section
   for (size_t i = 0; i < e_shnum; ++i) {
@@ -123,11 +105,14 @@ struct VDSOSymbolTable {
   ElfW(Half) * versym;
   Verdef *verdef;
 
-  void populate_symbol_cache(size_t symbol_count, ElfW(Addr) vdso_addr) {
+  void populate_symbol_cache(Symbol::VDSOArray &symbol_table,
+                             size_t symbol_count, ElfW(Addr) vdso_addr) {
     for (size_t i = 0, e = symbol_table.size(); i < e; ++i) {
       Symbol sym = i;
       cpp::string_view name = sym.name();
       cpp::string_view version = sym.version();
+      if (name.empty())
+        return;
       for (size_t j = 0; j < symbol_count; ++j) {
         if (name == strtab + symtab[j].st_name) {
           // we find a symbol with desired name
@@ -200,10 +185,11 @@ struct PhdrInfo {
     return VDSOSymbolTable{strtab, symtab, versym, verdef};
   }
 };
+} // namespace
 
-void initialize_vdso_global_cache() {
+void Symbol::initialize_vdso_global_cache() {
   // first clear the symbol table
-  for (auto &i : symbol_table)
+  for (auto &i : global_cache)
     i = nullptr;
 
   // get the address of the VDSO, protect errno since getauxval may change
@@ -249,20 +235,8 @@ void initialize_vdso_global_cache() {
     return;
 
   // finally, populate the global symbol table cache
-  vdso_symbol_table->populate_symbol_cache(symbol_count, phdr_info->vdso_addr);
-}
-} // namespace
-
-void *get_symbol(VDSOSym sym) {
-  // if sym is invalid, return nullptr
-  Symbol symbol(sym);
-  if (symbol >= symbol_table.size())
-    return nullptr;
-
-  static FutexWordType once_flag = 0;
-  callonce(reinterpret_cast<CallOnceFlag *>(&once_flag),
-           initialize_vdso_global_cache);
-  return symbol_table[symbol];
+  vdso_symbol_table->populate_symbol_cache(global_cache, symbol_count,
+                                           phdr_info->vdso_addr);
 }
 } // namespace vdso
 } // namespace LIBC_NAMESPACE

libc/src/__support/OSUtil/linux/vdso.h

@@ -7,8 +7,28 @@
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H
 #define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_VDSO_H
+#include "src/__support/CPP/array.h"
 #include "src/__support/common.h"
+#include "src/__support/macros/attributes.h"
 #include "src/__support/macros/properties/architectures.h"
+#include "src/__support/threads/callonce.h"
+
+namespace LIBC_NAMESPACE {
+namespace vdso {
+enum class VDSOSym {
+  ClockGetTime,
+  ClockGetTime64,
+  GetTimeOfDay,
+  GetCpu,
+  Time,
+  ClockGetRes,
+  RTSigReturn,
+  FlushICache,
+  RiscvHwProbe,
+  VDSOSymCount
+};
+} // namespace vdso
+} // namespace LIBC_NAMESPACE
 
 #if defined(LIBC_TARGET_ARCH_IS_X86)
 #include "x86_64/vdso.h"
@@ -24,7 +44,44 @@
 
 namespace LIBC_NAMESPACE {
 namespace vdso {
-void *get_symbol(VDSOSym);
+class Symbol {
+  VDSOSym sym;
+
+public:
+  LIBC_INLINE_VAR static constexpr size_t COUNT =
+      static_cast<size_t>(VDSOSym::VDSOSymCount);
+  LIBC_INLINE constexpr explicit Symbol(VDSOSym sym) : sym(sym) {}
+  LIBC_INLINE constexpr Symbol(size_t idx) : sym(static_cast<VDSOSym>(idx)) {}
+  LIBC_INLINE constexpr cpp::string_view name() const {
+    return symbol_name(sym);
+  }
+  LIBC_INLINE constexpr cpp::string_view version() const {
+    return symbol_version(sym);
+  }
+  LIBC_INLINE constexpr operator size_t() const {
+    return static_cast<size_t>(sym);
+  }
+  LIBC_INLINE constexpr bool is_valid() const {
+    return *this < Symbol::global_cache.size();
+  }
+  using VDSOArray = cpp::array<void *, Symbol::COUNT>;
+
+private:
+  static CallOnceFlag once_flag;
+  static VDSOArray global_cache;
+  static void initialize_vdso_global_cache();
+  friend void *get_symbol(VDSOSym sym);
+
+public:
+  template <typename T> LIBC_INLINE T get() {
+    if (name().empty() || !is_valid())
+      return nullptr;
+
+    callonce(&once_flag, Symbol::initialize_vdso_global_cache);
+    return cpp::bit_cast<T>(global_cache[*this]);
+  }
+};
+
 } // namespace vdso
 
 } // namespace LIBC_NAMESPACE

libc/src/__support/OSUtil/linux/x86_64/vdso.h

@@ -10,25 +10,6 @@
 #include "src/__support/CPP/string_view.h"
 namespace LIBC_NAMESPACE {
 namespace vdso {
-// macro definitions
-// following the order in
-// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/entry/vdso/vdso.lds.S
-#define LIBC_VDSO_HAS_CLOCK_GETTIME
-#define LIBC_VDSO_HAS_GETTIMEOFDAY
-#define LIBC_VDSO_HAS_GETCPU
-#define LIBC_VDSO_HAS_TIME
-#define LIBC_VDSO_HAS_CLOCK_GETRES
-
-// list of VDSO symbols
-enum class VDSOSym {
-  ClockGetTime,
-  GetTimeOfDay,
-  GetCpu,
-  Time,
-  ClockGetRes,
-  VDSOSymCount
-};
-
 // translate VDSOSym to symbol names
 // On x86, there are symbols defined without the __vdso_ prefix, however,
 // it is suggested that one should use the __vdso_ prefix.

libc/src/__support/threads/callonce.h

@@ -9,13 +9,38 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 
+#include "src/__support/macros/optimization.h"
+
+#ifdef __linux__
+#include "src/__support/threads/linux/futex_utils.h"
+#else
+#error "callonce is not supported on this platform"
+#endif
+
 namespace LIBC_NAMESPACE {
 
-struct CallOnceFlag;
+#ifdef __linux__
+using CallOnceFlag = Futex;
+#endif
 using CallOnceCallback = void(void);
+namespace callonce_impl {
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
+} // namespace callonce_impl
 
-int callonce(CallOnceFlag *flag, CallOnceCallback *callback);
+LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
+  using namespace callonce_impl;
+  // Avoid cmpxchg operation if the function has already been called.
+  // The destination operand of cmpxchg may receive a write cycle without
+  // regard to the result of the comparison
+  if (LIBC_LIKELY(flag->load(cpp::MemoryOrder::RELAXED) == FINISH))
+    return 0;
 
+  return callonce_slowpath(flag, callback);
+}
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H

libc/src/__support/threads/linux/callonce.cpp

@@ -7,27 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/threads/callonce.h"
-#include "src/__support/macros/optimization.h"
 #include "src/__support/threads/linux/futex_utils.h"
 
 namespace LIBC_NAMESPACE {
-
-static constexpr FutexWordType NOT_CALLED = 0x0;
-static constexpr FutexWordType START = 0x11;
-static constexpr FutexWordType WAITING = 0x22;
-static constexpr FutexWordType FINISH = 0x33;
-
-int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) {
   auto *futex_word = reinterpret_cast<Futex *>(flag);
 
   FutexWordType not_called = NOT_CALLED;
 
-  // Avoid cmpxchg operation if the function has already been called.
-  // The destination operand of cmpxchg may receive a write cycle without
-  // regard to the result of the comparison
-  if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH))
-    return 0;
-
   // The call_once call can return only after the called function |func|
   // returns. So, we use futexes to synchronize calls with the same flag value.
   if (futex_word->compare_exchange_strong(not_called, START)) {
@@ -46,5 +34,5 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
 
   return 0;
 }
-
+} // namespace callonce_impl
 } // namespace LIBC_NAMESPACE