Merge pull request #4643 from swiftwasm/main

[pull] swiftwasm from main

Author: kateinoigakukun

docs/OpenBSD.md

@@ -1,13 +1,13 @@
 # Getting started with Swift on OpenBSD
 
-Swift builds and runs on OpenBSD (tested on 6.8), with some special considerations.
+Swift builds and runs on OpenBSD, with some special considerations. This document was last tested on OpenBSD 7.1, but may also work on later versions.
 
 ## Preparing
 
 The following packages are required to build Swift. You can install these via `pkg_add`:
 
 ```shell
-$ doas pkg_add bash cmake e2fsprogs git icu4c ninja py-six python3
+$ doas pkg_add bash cmake e2fsprogs git icu4c ninja py3-six python3
 ```
 
 Because LLVM is built as part of building Swift and does not include some of the patches to handle the OpenBSD library naming convention, you will need to create some symlinks:
@@ -23,10 +23,10 @@ $ doas ln -s /usr/lib/libpthread.so.26.1 /usr/lib/libpthread.so
 
 *Note: you may need to update the version numbers if necessary.*
 
-Also link `~/bin/python` to the `python2.7` binary:
+Also link `~/bin/python` to the `python3` binary:
 
 ```shell
-$ doas ln -s /usr/local/bin/python2.7 ~/bin/python
+$ ln -s /usr/local/bin/python3 ~/bin/python
 ```
 
 Since the build requires significant amounts of memory at certain points, you may need to ensure that the user you are using to build Swift has the appropriate limits set. Using the `staff` group in `login.conf` and ensuring the shell limits are raised is recommended.
@@ -50,14 +50,16 @@ Download the sources with the [Getting Started](/docs/HowToGuides/GettingStarted
       "aliases": [ "main", "swift/main" ],
       "repos": {
         "cmark": "main",
-        "llvm-project": "swift/main",
+        "llvm-project": "stable/20211026",
         "swift": "main"
       }
     }
   }
 }
 ```
 
+*Note: you may need to check `utils/update_checkout/update-checkout-config.json` for the correct LLVM stable branch to build against.*
+
 ## Building
 
 Once the sources have completed downloading, you can use the standard `build-script` mechanism to start building Swift. However, some options are required to be set to successfully build Swift.
@@ -66,23 +68,26 @@ These options are:
 * `--skip-build-clang-tools-extra` and `--skip-build-compiler-rt`: to ensure LLVM builds cleanly,
 * `--extra-cmake-options=`
   * `-DCMAKE_DISABLE_FIND_PACKAGE_Backtrace=TRUE,-DCMAKE_DISABLE_FIND_PACKAGE_LibXml2=TRUE,-DLLVM_VERSION_SUFFIX=''`: to ensure LLVM builds cleanly,
-  * `-DSWIFT_BUILD_SOURCEKIT=OFF,-DSWIFT_BUILD_SYNTAXPARSERLIB=OFF,-DSWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY=OFF`: to ensure Swift does not attempt to build libdispatch, which is not yet supported on OpenBSD,
+  * `-DSWIFT_ENABLE_DISPATCH=FALSE,-DSWIFT_BUILD_SOURCEKIT=OFF,-DSWIFT_BUILD_SYNTAXPARSERLIB=OFF,-DSWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY=OFF,-DSWIFT_IMPLICIT_CONCURRENCY_IMPORT=OFF,-DSWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED=OFF`: to ensure Swift does not attempt to build libdispatch, which is not yet supported on OpenBSD,
   * `-DSWIFT_USE_LINKER=lld`: to specify that `lld` should be used over `gold`,
   * `-DCMAKE_INSTALL_DIR=/usr/local"`: to set the correct platform install directory.
 
 In full, the minimal set of flags to supply to `build-script` looks like:
 ```shell
 $ ./utils/build-script \
+    --release \
     --skip-build-clang-tools-extra \
     --skip-build-compiler-rt \
     --extra-cmake-options="\
         -DCMAKE_DISABLE_FIND_PACKAGE_Backtrace=TRUE,\
         -DCMAKE_DISABLE_FIND_PACKAGE_LibXml2=TRUE,\
         -DLLVM_VERSION_SUFFIX='',\
+        -DSWIFT_ENABLE_DISPATCH=OFF,\
         -DSWIFT_BUILD_SOURCEKIT=OFF,\
         -DSWIFT_BUILD_SYNTAXPARSERLIB=OFF,\
         -DSWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY=OFF,\
         -DSWIFT_IMPLICIT_CONCURRENCY_IMPORT=OFF,\
+        -DSWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED=OFF,\
         -DSWIFT_USE_LINKER=lld,\
         -DCMAKE_INSTALL_DIR=/usr/local"
 ```

docs/SIL.rst

@@ -3745,6 +3745,8 @@ store_borrow
 
 Stores the value ``%0`` to a stack location ``%1``, which must be an
 ``alloc_stack $T``.
+The stack location must not be modified by other instructions than
+``store_borrow``.
 The stored value is alive until the ``dealloc_stack`` or until another
 ``store_borrow`` overwrites the value. During the its lifetime, the stored
 value must not be modified or destroyed.

lib/SIL/Verifier/LoadBorrowImmutabilityChecker.cpp

@@ -363,9 +363,9 @@ bool LoadBorrowImmutabilityAnalysis::isImmutable(LoadBorrowInst *lbi) {
   // cases to try and exhaustively identify if those writes overlap with our
   // load_borrow.
   SmallVector<Operand *, 8> endBorrowUses;
-  transform(lbi->getUsersOfType<EndBorrowInst>(),
-            std::back_inserter(endBorrowUses),
-            [](EndBorrowInst *ebi) { return &ebi->getAllOperands()[0]; });
+  visitTransitiveEndBorrows(lbi, [&](EndBorrowInst *endBorrow) {
+    endBorrowUses.push_back(&endBorrow->getOperandRef());
+  });
 
   switch (accessPath.getStorage().getKind()) {
   case AccessStorage::Nested: {

lib/SIL/Verifier/MemoryLifetimeVerifier.cpp

@@ -89,6 +89,9 @@ class MemoryLifetimeVerifier {
   /// Register the destination address of a store_borrow as borrowed location.
   void registerStoreBorrowLocation(SILValue addr);
 
+  /// Registers all store_borrow instructions in a block.
+  void registerStoreBorrowsInBlock(SILBasicBlock *block);
+
   /// Handles locations of the predecessor's terminator, which are only valid
   /// in \p block.
   /// Example: @out results of try_apply. They are only valid in the
@@ -286,6 +289,13 @@ void MemoryLifetimeVerifier::registerStoreBorrowLocation(SILValue addr) {
   }
 }
 
+void MemoryLifetimeVerifier::registerStoreBorrowsInBlock(SILBasicBlock *block) {
+  for (SILInstruction &inst : *block) {
+    if (auto *sbi = dyn_cast<StoreBorrowInst>(&inst))
+      registerStoreBorrowLocation(sbi->getDest());
+  }
+}
+
 void MemoryLifetimeVerifier::initDataflow(BitDataflow &dataFlow) {
   // Initialize the entry and exit sets to all-bits-set. Except for the function
   // entry.
@@ -579,7 +589,6 @@ void MemoryLifetimeVerifier::checkBlock(SILBasicBlock *block, Bits &bits) {
       case SILInstructionKind::StoreBorrowInst: {
         SILValue destAddr = cast<StoreBorrowInst>(&I)->getDest();
         locations.setBits(bits, destAddr);
-        registerStoreBorrowLocation(destAddr);
         break;
       }
       case SILInstructionKind::CopyAddrInst: {
@@ -764,6 +773,7 @@ void MemoryLifetimeVerifier::verify() {
   locations.handleSingleBlockLocations([this](SILBasicBlock *block) {
     storeBorrowLocations.clear();
     Bits bits(locations.getNumLocations());
+    registerStoreBorrowsInBlock(block);
     checkBlock(block, bits);
   });
 }

lib/Threading/Pthreads.cpp

@@ -71,7 +71,7 @@ void swift::threading_impl::once_slow(once_t &predicate, void (*fn)(void *),
 
   pthread_mutex_lock(&onceMutex);
   while (predicate.load(std::memory_order_acquire) >= (std::int64_t)0) {
-    pthread_cond_wait(&onceCond);
+    pthread_cond_wait(&onceCond, &onceMutex);
   }
   pthread_mutex_unlock(&onceMutex);
 }

stdlib/public/Concurrency/Task.cpp

@@ -113,6 +113,7 @@ FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask,
 
   auto queueHead = fragment->waitQueue.load(std::memory_order_acquire);
   bool contextInitialized = false;
+  auto escalatedPriority = JobPriority::Unspecified;
   while (true) {
     switch (queueHead.getStatus()) {
     case Status::Error:
@@ -145,6 +146,47 @@ FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask,
       waitingTask->flagAsSuspended();
     }
 
+    // Escalate the blocking task to the priority of the waiting task.
+    // FIXME: Also record that the waiting task is now waiting on the
+    // blocking task so that escalators of the waiting task can propagate
+    // the escalation to the blocking task.
+    //
+    // Recording this dependency is tricky because we need escalators
+    // to be able to escalate without worrying about the blocking task
+    // concurrently finishing, resuming the escalated task, and being
+    // invalidated.  So we're not doing that yet.  In the meantime, we
+    // do the best-effort alternative of escalating the blocking task
+    // as a one-time deal to the current priority of the waiting task.
+    // If the waiting task is escalated after this point, the priority
+    // will not be escalated, but that's inevitable in the absence of
+    // propagation during escalation.
+    //
+    // We have to do the escalation before we successfully enqueue the
+    // waiting task on the blocking task's wait queue, because as soon as
+    // we do, this thread is no longer blocking the resumption of the
+    // waiting task, and so both the blocking task (which is retained
+    // during the wait only from the waiting task's perspective) and the
+    // waiting task (which can simply terminate) must be treat as
+    // invalidated from this thread's perspective.
+    //
+    // When we do fix this bug to record the dependency, we will have to
+    // do it before this escalation of the blocking task so that there
+    // isn't a race where an escalation of the waiting task can fail
+    // to propagate to the blocking task.  The correct priority to
+    // escalate to is the priority we observe when we successfully record
+    // the dependency; any later escalations will automatically propagate.
+    //
+    // If the blocking task finishes while we're doing this escalation,
+    // the escalation will be innocuous.  The wasted effort is acceptable;
+    // programmers should be encouraged to give tasks that will block
+    // other tasks the correct priority to begin with.
+    auto waitingStatus =
+      waitingTask->_private()._status().load(std::memory_order_relaxed);
+    if (waitingStatus.getStoredPriority() > escalatedPriority) {
+      swift_task_escalate(this, waitingStatus.getStoredPriority());
+      escalatedPriority = waitingStatus.getStoredPriority();
+    }
+
     // Put the waiting task at the beginning of the wait queue.
     waitingTask->getNextWaitingTask() = queueHead.getTask();
     auto newQueueHead = WaitQueueItem::get(Status::Executing, waitingTask);
@@ -153,11 +195,6 @@ FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask,
             /*success*/ std::memory_order_release,
             /*failure*/ std::memory_order_acquire)) {
 
-      // Escalate the priority of this task based on the priority
-      // of the waiting task.
-      auto status = waitingTask->_private()._status().load(std::memory_order_relaxed);
-      swift_task_escalate(this, status.getStoredPriority());
-
       _swift_task_clearCurrent();
       return FutureFragment::Status::Executing;
     }

stdlib/public/stubs/Random.cpp

@@ -39,6 +39,10 @@
 
 #endif
 
+#if __has_include(<unistd.h>)
+#include <unistd.h>
+#endif
+
 #include <stdlib.h>
 #include <unistd.h>
 

test/SIL/ownership-verifier/arguments.sil renamed to test/SIL/OwnershipVerifier/arguments.sil

@@ -153,3 +153,23 @@ bb0(%0 : @owned $K):
   return %r : $()
 }
 
+sil @use_guaranteed : $@convention(thin) (@guaranteed K) -> ()
+
+sil [ossa] @test_write_reborrow : $@convention(thin) (@owned K, @owned K) -> () {
+bb0(%0 : @owned $K, %1 : @owned $K):
+  %stk = alloc_stack [lexical] $K
+  store %0 to [init] %stk : $*K
+  %ld1 = load_borrow %stk : $*K
+  br bb2(%ld1 : $K)
+
+bb2(%ld : @guaranteed $K):
+  %3 = function_ref @use_guaranteed : $@convention(thin) (@guaranteed K) -> ()
+  %4 = apply %3(%ld) : $@convention(thin) (@guaranteed K) -> ()
+  end_borrow %ld : $K
+  store %1 to [assign] %stk : $*K
+  destroy_addr %stk : $*K
+  dealloc_stack %stk : $*K
+  %6 = tuple ()
+  return %6 : $()
+}
+

test/SIL/ownership-verifier/begin_apply_use_after_end_apply.sil renamed to test/SIL/OwnershipVerifier/begin_apply_use_after_end_apply.sil

@@ -0,0 +1,27 @@
+// RUN: %target-sil-opt %s -verify-continue-on-failure=true -o /dev/null 2>&1 | %FileCheck %s
+
+class Klass {}
+
+sil @use_guaranteed : $@convention(thin) (@guaranteed Klass) -> ()
+
+// Write:   store {{.*}} [assign] {{.*}}
+// CHECK: Begin Error in function test_write_reborrow
+// CHECK: SIL verification failed: Found load borrow that is invalidated by a local write?!: loadBorrowImmutabilityAnalysis.isImmutable(LBI)
+// CHECK: End Error in function test_write_reborrow
+sil [ossa] @test_write_reborrow : $@convention(thin) (@owned Klass, @owned Klass) -> () {
+bb0(%0 : @owned $Klass, %1 : @owned $Klass):
+  %stk = alloc_stack [lexical] $Klass
+  store %0 to [init] %stk : $*Klass
+  %ld1 = load_borrow %stk : $*Klass
+  br bb2(%ld1 : $Klass)
+
+bb2(%ld : @guaranteed $Klass):
+  store %1 to [assign] %stk : $*Klass
+  %3 = function_ref @use_guaranteed : $@convention(thin) (@guaranteed Klass) -> ()
+  %4 = apply %3(%ld) : $@convention(thin) (@guaranteed Klass) -> ()
+  end_borrow %ld : $Klass
+  destroy_addr %stk : $*Klass
+  dealloc_stack %stk : $*Klass
+  %6 = tuple ()
+  return %6 : $()
+}

test/SIL/ownership-verifier/borrow_and_concurrency.sil renamed to test/SIL/OwnershipVerifier/borrow_and_concurrency.sil

@@ -361,6 +361,36 @@ bb0(%0 :  @guaranteed $Optional<T>):
   return %res : $()
 }
 
+// CHECK: SIL memory lifetime failure in @test_store_borrow_single_block: store-borrow location cannot be written
+sil [ossa] @test_store_borrow_single_block : $@convention(thin) (@guaranteed T) -> () {
+bb0(%0 : @guaranteed $T):
+  %stk = alloc_stack $T
+  %copy = copy_value %0 : $T
+  store %copy to [init] %stk : $*T
+  %ld = load [take] %stk : $*T
+  destroy_value %ld : $T
+  store_borrow %0 to %stk : $*T
+  dealloc_stack %stk : $*T
+  %8 = tuple ()
+  return %8 : $() 
+}
+
+// CHECK: SIL memory lifetime failure in @test_store_borrow_multi_block: store-borrow location cannot be written
+sil [ossa] @test_store_borrow_multi_block : $@convention(thin) (@guaranteed T) -> () {
+bb0(%0 : @guaranteed $T):
+  %stk = alloc_stack $T
+  %copy = copy_value %0 : $T
+  store %copy to [init] %stk : $*T
+  %ld = load [take] %stk : $*T
+  destroy_value %ld : $T
+  br bb1
+bb1:
+  store_borrow %0 to %stk : $*T
+  dealloc_stack %stk : $*T
+  %8 = tuple ()
+  return %8 : $() 
+}
+
 // CHECK: SIL memory lifetime failure in @test_cast_br_take_always: memory is not initialized, but should be
 sil [ossa] @test_cast_br_take_always : $@convention(thin) <U, V> (@in U) -> () {
 bb0(%0 : $*U):