Merge pull request #19289 from mikeash/thread-local-storage-dedicated-keys

[Runtime][Stdlib] Use dedicated thread-local storage keys when possible.

Author: mikeash

stdlib/public/SwiftShims/CMakeLists.txt

@@ -13,6 +13,7 @@ set(sources
   SwiftStddef.h
   SwiftStdint.h
   System.h
+  ThreadLocalStorage.h
   UnicodeShims.h
   Visibility.h
 

stdlib/public/SwiftShims/LibcShims.h

@@ -189,35 +189,6 @@ double lgamma_r(double x, int *psigngam);
 long double lgammal_r(long double x, int *psigngam);
 #endif // defined(__APPLE__)
 
-// TLS - thread local storage
-
-#if defined(__linux__)
-# if defined(__ANDROID__)
-typedef int __swift_thread_key_t;
-# else
-typedef unsigned int __swift_thread_key_t;
-# endif
-#elif defined(__FreeBSD__)
-typedef int __swift_thread_key_t;
-#elif defined(_WIN32)
-typedef unsigned long __swift_thread_key_t;
-#elif defined(__HAIKU__)
-typedef int __swift_thread_key_t;
-#else
-typedef unsigned long __swift_thread_key_t;
-#endif
-
-SWIFT_RUNTIME_STDLIB_INTERNAL
-int _stdlib_thread_key_create(__swift_thread_key_t * _Nonnull key,
-                              void (* _Nullable destructor)(void * _Nullable));
-
-SWIFT_RUNTIME_STDLIB_INTERNAL
-void * _Nullable _stdlib_thread_getspecific(__swift_thread_key_t key);
-
-SWIFT_RUNTIME_STDLIB_INTERNAL
-int _stdlib_thread_setspecific(__swift_thread_key_t key,
-                               const void * _Nullable value);
-
 #ifdef __cplusplus
 }} // extern "C", namespace swift
 #endif

stdlib/public/SwiftShims/ThreadLocalStorage.h

@@ -0,0 +1,21 @@
+//===--- ThreadLocalStorage.h - Wrapper for thread-local storage. --*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_STDLIB_SHIMS_THREADLOCALSTORAGE_H
+#define SWIFT_STDLIB_SHIMS_THREADLOCALSTORAGE_H
+
+#include "Visibility.h"
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+void * _Nonnull _swift_stdlib_threadLocalStorageGet(void);
+
+#endif // SWIFT_STDLIB_SHIMS_THREADLOCALSTORAGE_H

stdlib/public/SwiftShims/module.modulemap

@@ -13,6 +13,7 @@ module SwiftShims {
   header "SwiftStddef.h"
   header "SwiftStdint.h"
   header "System.h"
+  header "ThreadLocalStorage.h"
   header "UnicodeShims.h"
   header "Visibility.h"
   export *

stdlib/public/core/ThreadLocalStorage.swift

@@ -64,13 +64,8 @@ internal struct _ThreadLocalStorage {
   internal static func getPointer()
     -> UnsafeMutablePointer<_ThreadLocalStorage>
   {
-    let tlsRawPtr = _stdlib_thread_getspecific(_tlsKey)
-    if _fastPath(tlsRawPtr != nil) {
-      return tlsRawPtr._unsafelyUnwrappedUnchecked.assumingMemoryBound(
-        to: _ThreadLocalStorage.self)
-    }
-
-    return _initializeThreadLocalStorage()
+    return _swift_stdlib_threadLocalStorageGet().assumingMemoryBound(
+      to: _ThreadLocalStorage.self)
   }
 
   internal static func getUBreakIterator(
@@ -105,23 +100,10 @@ internal func _destroyTLS(_ ptr: UnsafeMutableRawPointer?) {
 #endif
 }
 
-// Lazily created global key for use with pthread TLS
-internal let _tlsKey: __swift_thread_key_t = {
-  let sentinelValue = __swift_thread_key_t.max
-  var key: __swift_thread_key_t = sentinelValue
-  let success = _stdlib_thread_key_create(&key, _destroyTLS)
-  _sanityCheck(success == 0, "somehow failed to create TLS key")
-  _sanityCheck(key != sentinelValue, "Didn't make a new key")
-  return key
-}()
-
-@inline(never)
-internal func _initializeThreadLocalStorage()
+@_silgen_name("_stdlib_createTLS")
+internal func _createThreadLocalStorage()
   -> UnsafeMutablePointer<_ThreadLocalStorage>
 {
-  _sanityCheck(_stdlib_thread_getspecific(_tlsKey) == nil,
-    "already initialized")
-
   // Create and initialize one.
   var err = __swift_stdlib_U_ZERO_ERROR
   let newUBreakIterator = __swift_stdlib_ubrk_open(
@@ -136,7 +118,5 @@ internal func _initializeThreadLocalStorage()
   tlsPtr.initialize(
     to: _ThreadLocalStorage(_uBreakIterator: newUBreakIterator)
   )
-  let success = _stdlib_thread_setspecific(_tlsKey, tlsPtr)
-  _sanityCheck(success == 0, "setspecific failed")
   return tlsPtr
 }

stdlib/public/runtime/Exclusivity.cpp

@@ -14,43 +14,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "swift/Basic/Lazy.h"
 #include "swift/Runtime/Config.h"
 #include "swift/Runtime/Debug.h"
 #include "swift/Runtime/Exclusivity.h"
 #include "swift/Runtime/Metadata.h"
+#include "ThreadLocalStorage.h"
 #include <memory>
 #include <stdio.h>
 
-// Pick an implementation strategy.
-#ifndef SWIFT_EXCLUSIVITY_USE_THREADLOCAL
-
-// If we're using Clang, and Clang claims not to support thread_local,
-// it must be because we're on a platform that doesn't support it.
-// Use pthreads.
-// Workaround: has_feature(cxx_thread_local) is wrong on two old Apple
-// simulators. clang thinks thread_local works there, but it doesn't.
-#if TARGET_OS_SIMULATOR && !TARGET_RT_64_BIT &&                      \
-  ((TARGET_OS_IOS && __IPHONE_OS_VERSION_MIN_REQUIRED__ < 100000) || \
-   (TARGET_OS_WATCH && __WATCHOS_OS_VERSION_MIN_REQUIRED__ < 30000))
-// 32-bit iOS 9 simulator or 32-bit watchOS 2 simulator - use pthreads
-# define SWIFT_EXCLUSIVITY_USE_THREADLOCAL 0
-# define SWIFT_EXCLUSIVITY_USE_PTHREAD_SPECIFIC 1
-#elif __clang__ && !__has_feature(cxx_thread_local)
-// clang without thread_local support - use pthreads
-# define SWIFT_EXCLUSIVITY_USE_THREADLOCAL 0
-# define SWIFT_EXCLUSIVITY_USE_PTHREAD_SPECIFIC 1
-#else
-// Use thread_local
-# define SWIFT_EXCLUSIVITY_USE_THREADLOCAL 1
-# define SWIFT_EXCLUSIVITY_USE_PTHREAD_SPECIFIC 0
-#endif
-
-#endif
-
-#if SWIFT_EXCLUSIVITY_USE_PTHREAD_SPECIFIC
-#include <pthread.h>
-#endif
-
 // Pick a return-address strategy
 #if __GNUC__
 #define get_return_address() __builtin_return_address(0)
@@ -277,46 +249,63 @@ class AccessSet {
 // Each of these cases should define a function with this prototype:
 //   AccessSets &getAllSets();
 
-#if SWIFT_EXCLUSIVITY_USE_THREADLOCAL
-// Use direct language support for thread-locals.
+#if SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC
+// Use the reserved TSD key if possible.
+
+static AccessSet &getAccessSet() {
+  AccessSet *set = static_cast<AccessSet*>(
+    SWIFT_THREAD_GETSPECIFIC(SWIFT_EXCLUSIVITY_TLS_KEY));
+  if (set)
+    return *set;
+  
+  static OnceToken_t setupToken;
+  SWIFT_ONCE_F(setupToken, [](void *) {
+    pthread_key_init_np(SWIFT_EXCLUSIVITY_TLS_KEY, [](void *pointer) {
+      delete static_cast<AccessSet*>(pointer);
+    });
+  }, nullptr);
+  
+  set = new AccessSet();
+  SWIFT_THREAD_SETSPECIFIC(SWIFT_EXCLUSIVITY_TLS_KEY, set);
+  return *set;
+}
+
+#elif SWIFT_TLS_HAS_THREADLOCAL
+// Second choice is direct language support for thread-locals.
 
-static_assert(LLVM_ENABLE_THREADS, "LLVM_THREAD_LOCAL will use a global?");
 static LLVM_THREAD_LOCAL AccessSet ExclusivityAccessSet;
 
 static AccessSet &getAccessSet() {
   return ExclusivityAccessSet;
 }
 
-#elif SWIFT_EXCLUSIVITY_USE_PTHREAD_SPECIFIC
-// Use pthread_getspecific.
+#else
+// Use the platform thread-local data API.
 
-static pthread_key_t createAccessSetPthreadKey() {
-  pthread_key_t key;
-  int result = pthread_key_create(&key, [](void *pointer) {
+static __swift_thread_key_t createAccessSetThreadKey() {
+  __swift_thread_key_t key;
+  int result = SWIFT_THREAD_KEY_CREATE(&key, [](void *pointer) {
     delete static_cast<AccessSet*>(pointer);
   });
 
   if (result != 0) {
-    fatalError(0, "couldn't create pthread key for exclusivity: %s\n",
+    fatalError(0, "couldn't create thread key for exclusivity: %s\n",
                strerror(result));
   }
   return key;
 }
 
 static AccessSet &getAccessSet() {
-  static pthread_key_t key = createAccessSetPthreadKey();
+  static __swift_thread_key_t key = createAccessSetThreadKey();
 
-  AccessSet *set = static_cast<AccessSet*>(pthread_getspecific(key));
+  AccessSet *set = static_cast<AccessSet*>(SWIFT_THREAD_GETSPECIFIC(key));
   if (!set) {
     set = new AccessSet();
-    pthread_setspecific(key, set);
+    SWIFT_THREAD_SETSPECIFIC(key, set);
   }
   return *set;
 }
 
-/** An access set accessed via pthread_get_specific. *************************/
-#else
-#error No implementation chosen for exclusivity!
 #endif
 
 /// Begin tracking a dynamic access.

stdlib/public/runtime/ThreadLocalStorage.h

@@ -0,0 +1,117 @@
+//===--- ThreadLocalStorage.h - Thread-local storage interface. --*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_RUNTIME_THREADLOCALSTORAGE_H
+#define SWIFT_RUNTIME_THREADLOCALSTORAGE_H
+
+#include "llvm/Support/Compiler.h"
+#include "swift/Runtime/Config.h"
+
+// Depending on the target, we may be able to use dedicated TSD keys or
+// thread_local variables. When dedicated TSD keys aren't available,
+// wrap the target's API for thread-local data for things that don't want
+// to use thread_local.
+
+// On Apple platforms, we have dedicated TSD keys.
+#if defined(__APPLE__)
+# define SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC 1
+#endif
+
+// If we're using Clang, and Clang claims not to support thread_local,
+// it must be because we're on a platform that doesn't support it.
+#if __clang__ && !__has_feature(cxx_thread_local)
+// No thread_local.
+#else
+// Otherwise, we do have thread_local.
+# define SWIFT_TLS_HAS_THREADLOCAL 1
+static_assert(LLVM_ENABLE_THREADS, "LLVM_THREAD_LOCAL will use a global?");
+#endif
+
+#if SWIFT_TLS_HAS_RESERVED_PTHREAD_SPECIFIC
+// Use reserved TSD keys.
+# if __has_include(<pthread/tsd_private.h>)
+#  include <pthread/tsd_private.h>
+# else
+// We still need to use the SPI for setting the destructor, so declare it here.
+extern "C" int pthread_key_init_np(int key, void (*destructor)(void *));
+# endif
+
+// If the keys are not available from the header, define them ourselves. The values match
+// what tsd_private.h provides.
+# ifndef __PTK_FRAMEWORK_SWIFT_KEY0
+#  define __PTK_FRAMEWORK_SWIFT_KEY0 100
+# endif
+# ifndef __PTK_FRAMEWORK_SWIFT_KEY1
+#  define __PTK_FRAMEWORK_SWIFT_KEY1 101
+# endif
+
+# define SWIFT_EXCLUSIVITY_TLS_KEY __PTK_FRAMEWORK_SWIFT_KEY0
+# define SWIFT_STDLIB_TLS_KEY __PTK_FRAMEWORK_SWIFT_KEY1
+
+#endif
+
+// If the reserved key path didn't already provide get/setspecific macros,
+// wrap the platform's APIs.
+#ifndef SWIFT_THREAD_GETSPECIFIC
+
+// Pick the right typedef for the key.
+# if defined(__linux__)
+#  if defined(__ANDROID__)
+typedef int __swift_thread_key_t;
+#  else
+typedef unsigned int __swift_thread_key_t;
+#  endif
+# elif defined(__FreeBSD__)
+typedef int __swift_thread_key_t;
+# elif defined(_WIN32)
+typedef unsigned long __swift_thread_key_t;
+# elif defined(__HAIKU__)
+typedef int __swift_thread_key_t;
+# else
+typedef unsigned long __swift_thread_key_t;
+# endif
+
+# if defined(_WIN32) && !defined(__CYGWIN__)
+// Windows has its own flavor of API.
+#  include <io.h>
+#  define WIN32_LEAN_AND_MEAN
+#  include <Windows.h>
+static_assert(std::is_same<__swift_thread_key_t, DWORD>::value,
+              "__swift_thread_key_t is not a DWORD");
+
+#  if defined(_M_IX86)
+typedef stdcall void (*__swift_thread_key_destructor)(void *)
+#  else
+typedef void (*__swift_thread_key_destructor)(void *)
+#  endif
+
+static inline
+_stdlib_thread_key_create(__swift_thread_key_t * _Nonnull key,
+                          __swift_thread_key_destructor _Nullable destructor) {
+  *key = FlsAlloc(destroyTLS_CCAdjustmentThunk);
+  return *key != FLS_OUT_OF_INDEXES;
+}
+
+#  define SWIFT_THREAD_KEY_CREATE _stdlib_thread_key_create
+#  define SWIFT_THREAD_GETSPECIFIC FlsGetValue
+#  define SWIFT_THREAD_SETSPECIFIC(key, value) (FlsSetValue(key, value) == TRUE)
+
+# else
+// Otherwise use the pthread API.
+#  include <pthread.h>
+#  define SWIFT_THREAD_KEY_CREATE pthread_key_create
+#  define SWIFT_THREAD_GETSPECIFIC pthread_getspecific
+#  define SWIFT_THREAD_SETSPECIFIC pthread_setspecific
+# endif
+#endif
+
+#endif // SWIFT_RUNTIME_THREADLOCALSTORAGE_H

stdlib/public/stubs/CMakeLists.txt

@@ -5,6 +5,7 @@ set(swift_stubs_sources
     KeyPaths.cpp
     LibcShims.cpp
     Stubs.cpp
+    ThreadLocalStorage.cpp
     MathStubs.cpp
 )
 set(swift_stubs_objc_sources