CanonicalOSSALifetime: Add support for overlapping access scopes.

Access scopes for enforcing exclusivity are currently the only
exception to our ability to canonicalize OSSA lifetime purely based on
the SSA value's known uses. This is because access scopes have
semantics relative to object deinitializers.

In general, deinitializers are asynchronous with respect to code that
is unrelated to the object's uses. Ignoring exclusivity, the optimizer
may always destroy objects as early as it wants, as long as the object
won't be used again. The optimizer may also extend the lifetime
(although in the future this lifetime extension should be limited by
"synchronization points").

The optimizer's freedom is however limited by exclusivity
enforcement. Optimization may never introduce new exclusivity
violations. Destroying an object within an access scope is an
exclusivity violation if the deinitializer accesses the same variable.

To handle this, OSSA canonicalization must detect access scopes that
overlap with the end of the pruned extended lifetime. Essentially:

    %def
    begin_access // access scope unrelated to def
    use %def     // pruned liveness ends here
    end_access
    destroy %def

Support for access scopes composes cleanly with the existing algorithm
without adding significant cost in the usual case. Overlapping access
scopes are unusual. A single CFG walk within the original extended
lifetime is normally sufficient. Only the blocks that are not already
LiveOut in the pruned liveness need to be visited. During this walk,
local overlapping access are detected by scanning for end_access
instructions after the last use point. Global overlapping accesses are
detected by checking NonLocalAccessBlockAnalysis. This avoids scanning
instructions in the common case. NonLocalAccessBlockAnalysis is a
trivial analysis that caches the rare occurence of nonlocal access
scopes. The analysis itself is a single linear scan over the
instruction stream. This analysis can be preserved across most
transformations and I expect it to be used to speed up other
optimizations related to access marker.

When an overlapping access is detected, pruned liveness is simply
extended to include the end_access as a new use point. Extending the
lifetime is iterative, but with each iteration, blocks that are now
marked LiveOut no longer need to be visited. Furthermore, interleaved
accessed scopes are not expected to happen in practice.

Author: atrick

include/swift/Basic/Compiler.h

@@ -121,11 +121,11 @@
 //   }
 // };
 #ifdef NDEBUG
-#define SWIFT_ASSERT_ONLY_DECL(X)
-#define SWIFT_ASSERT_ONLY(X) do { } while (false)
+#define SWIFT_ASSERT_ONLY_DECL(...)
+#define SWIFT_ASSERT_ONLY(...) do { } while (false)
 #else
-#define SWIFT_ASSERT_ONLY_DECL(X) X
-#define SWIFT_ASSERT_ONLY(X) do { X; } while (false)
+#define SWIFT_ASSERT_ONLY_DECL(X...) X
+#define SWIFT_ASSERT_ONLY(X...) do { X; } while (false)
 #endif
 
 #endif // SWIFT_BASIC_COMPILER_H

include/swift/SILOptimizer/Analysis/Analysis.def

@@ -38,6 +38,7 @@ ANALYSIS(Escape)
 ANALYSIS(InductionVariable)
 ANALYSIS(Loop)
 ANALYSIS(LoopRegion)
+ANALYSIS(NonLocalAccessBlock)
 ANALYSIS(OptimizerStats)
 ANALYSIS(PostDominance)
 ANALYSIS(PostOrder)

include/swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h

@@ -0,0 +1,97 @@
+//===--- NonLocalAccessBlockAnalysis.h - Nonlocal end_access ----*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// Cache the set of blocks that contain a non-local end_access, which is a rare
+/// occurrence. Optimizations that are driven by a known-use analysis, such as
+/// CanonicalOSSA, don't need to scan instructions that are unrelated to the SSA
+/// def-use graph. However, they may still need to be aware of unrelated access
+/// scope boundaries. By querying this analysis, they can avoid scanning all
+/// instructions just to deal with the extremely rare case of an end_access that
+/// spans blocks within the relevant SSA lifetime.
+///
+/// By default, this analysis is invalidated whenever instructions or blocks are
+/// changed, but it should ideally be preserved by passes that invalidate
+/// instructions but don't create any new access scopes or move end_access
+/// across blocks, which is unusual.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SILOPTIMIZER_ANALYSIS_NONLOCALACCESSBLOCKS_H
+#define SWIFT_SILOPTIMIZER_ANALYSIS_NONLOCALACCESSBLOCKS_H
+
+#include "swift/Basic/Compiler.h"
+#include "swift/SILOptimizer/Analysis/Analysis.h"
+#include "llvm/ADT/SmallPtrSet.h"
+
+namespace swift {
+
+class SILBasicBlock;
+class SILFunction;
+
+class NonLocalAccessBlocks {
+  friend class NonLocalAccessBlockAnalysis;
+
+  SILFunction *function;
+  llvm::SmallPtrSet<SILBasicBlock *, 4> accessBlocks;
+
+public:
+  NonLocalAccessBlocks(SILFunction *function) : function(function) {}
+
+  SILFunction *getFunction() const { return function; }
+
+  bool containsNonLocalEndAccess(SILBasicBlock *block) const {
+    return accessBlocks.count(block);
+  }
+
+  /// Perform NonLocalAccessBlockAnalysis for this function. Populate
+  /// this->accessBlocks with all blocks containing a non-local end_access.
+  void compute();
+};
+
+class NonLocalAccessBlockAnalysis
+    : public FunctionAnalysisBase<NonLocalAccessBlocks> {
+public:
+  static bool classof(const SILAnalysis *S) {
+    return S->getKind() == SILAnalysisKind::NonLocalAccessBlock;
+  }
+  NonLocalAccessBlockAnalysis()
+      : FunctionAnalysisBase<NonLocalAccessBlocks>(
+          SILAnalysisKind::NonLocalAccessBlock) {}
+
+  NonLocalAccessBlockAnalysis(const NonLocalAccessBlockAnalysis &) = delete;
+
+  NonLocalAccessBlockAnalysis &
+  operator=(const NonLocalAccessBlockAnalysis &) = delete;
+
+protected:
+  virtual std::unique_ptr<NonLocalAccessBlocks>
+  newFunctionAnalysis(SILFunction *function) override {
+    auto result = std::make_unique<NonLocalAccessBlocks>(function);
+    result->compute();
+    return result;
+  }
+
+  virtual bool shouldInvalidate(SILAnalysis::InvalidationKind kind) override {
+    return kind & InvalidationKind::BranchesAndInstructions;
+  }
+
+  SWIFT_ASSERT_ONLY_DECL(
+      virtual void verify(NonLocalAccessBlocks *accessBlocks) const override {
+        NonLocalAccessBlocks checkAccessBlocks(accessBlocks->function);
+        checkAccessBlocks.compute();
+        assert(checkAccessBlocks.accessBlocks == accessBlocks->accessBlocks);
+      })
+};
+
+} // end namespace swift
+
+#endif

include/swift/SILOptimizer/Utils/CanonicalOSSALifetime.h

@@ -93,10 +93,12 @@
 #ifndef SWIFT_SILOPTIMIZER_UTILS_CANONICALOSSALIFETIME_H
 #define SWIFT_SILOPTIMIZER_UTILS_CANONICALOSSALIFETIME_H
 
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SetVector.h"
 #include "swift/SIL/SILInstruction.h"
+#include "swift/SILOptimizer/Analysis/DominanceAnalysis.h"
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Utils/PrunedLiveness.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SetVector.h"
 
 namespace swift {
 
@@ -187,6 +189,13 @@ class CanonicalizeOSSALifetime {
   /// liveness may be pruned during canonicalization.
   bool pruneDebug;
 
+  NonLocalAccessBlockAnalysis *accessBlockAnalysis;
+  // Lazilly initialize accessBlocks only when
+  // extendLivenessThroughOverlappingAccess is invoked.
+  NonLocalAccessBlocks *accessBlocks = nullptr;
+
+  DominanceAnalysis *dominanceAnalysis;
+
   /// Current copied def for which this state describes the liveness.
   SILValue currentDef;
 
@@ -208,7 +217,7 @@ class CanonicalizeOSSALifetime {
   /// Record all interesting debug_value instructions here rather then treating
   /// them like a normal use. An interesting debug_value is one that may lie
   /// outisde the pruned liveness at the time it is discovered.
-  llvm::SmallDenseSet<DebugValueInst *, 8> debugValues;
+  llvm::SmallPtrSet<DebugValueInst *, 8> debugValues;
 
   /// Reuse a general worklist for def-use traversal.
   SmallSetVector<SILValue, 8> defUseWorklist;
@@ -226,12 +235,19 @@ class CanonicalizeOSSALifetime {
   CanonicalOSSAConsumeInfo consumes;
 
 public:
-  CanonicalizeOSSALifetime(bool pruneDebug) : pruneDebug(pruneDebug) {}
+  CanonicalizeOSSALifetime(bool pruneDebug,
+                           NonLocalAccessBlockAnalysis *accessBlockAnalysis,
+                           DominanceAnalysis *dominanceAnalysis)
+    : pruneDebug(pruneDebug), accessBlockAnalysis(accessBlockAnalysis),
+      dominanceAnalysis(dominanceAnalysis) {}
 
   SILValue getCurrentDef() const { return currentDef; }
 
   void initDef(SILValue def) {
     assert(consumingBlocks.empty() && debugValues.empty() && liveness.empty());
+    // Clear the cached analysis pointer just in case the client invalidates the
+    // analysis, freeing its memory.
+    accessBlocks = nullptr;
     consumes.clear();
 
     currentDef = def;
@@ -281,6 +297,10 @@ class CanonicalizeOSSALifetime {
 
   bool computeCanonicalLiveness();
 
+  bool endsAccessOverlappingPrunedBoundary(SILInstruction *inst);
+
+  void extendLivenessThroughOverlappingAccess();
+
   void findOrInsertDestroyInBlock(SILBasicBlock *bb);
 
   void findOrInsertDestroys();

include/swift/SILOptimizer/Utils/PrunedLiveness.h

@@ -147,7 +147,7 @@ class PrunedLiveBlocks {
   }
 
   /// Update this liveness result for a single use.
-  IsLive updateForUse(Operand *use);
+  IsLive updateForUse(SILInstruction *user);
 
   IsLive getBlockLiveness(SILBasicBlock *bb) const {
     auto liveBlockIter = liveBlocks.find(bb);
@@ -217,7 +217,7 @@ class PrunedLiveness {
   /// relationships that generate liveness. For example, use->isLifetimeEnding()
   /// cannot distinguish the end of the borrow scope that defines this extended
   /// live range vs. a nested borrow scope within the extended live range.
-  void updateForUse(Operand *use, bool lifetimeEnding);
+  void updateForUse(SILInstruction *user, bool lifetimeEnding);
 
   PrunedLiveBlocks::IsLive getBlockLiveness(SILBasicBlock *bb) const {
     return liveBlocks.getBlockLiveness(bb);

lib/SILOptimizer/Analysis/Analysis.cpp

@@ -20,6 +20,7 @@
 #include "swift/SILOptimizer/Analysis/ClassHierarchyAnalysis.h"
 #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h"
 #include "swift/SILOptimizer/Analysis/IVAnalysis.h"
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Analysis/PostOrderAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ProtocolConformanceAnalysis.h"
 #include "swift/SILOptimizer/Utils/InstOptUtils.h"
@@ -60,3 +61,7 @@ SILAnalysis *swift::createBasicCalleeAnalysis(SILModule *M) {
 SILAnalysis *swift::createProtocolConformanceAnalysis(SILModule *M) {
   return new ProtocolConformanceAnalysis(M);
 }
+
+SILAnalysis *swift::createNonLocalAccessBlockAnalysis(SILModule *M) {
+  return new NonLocalAccessBlockAnalysis();
+}

lib/SILOptimizer/Analysis/CMakeLists.txt

@@ -19,6 +19,7 @@ target_sources(swiftSILOptimizer PRIVATE
   LoopAnalysis.cpp
   LoopRegionAnalysis.cpp
   MemoryBehavior.cpp
+  NonLocalAccessBlockAnalysis.cpp
   PassManagerVerifierAnalysis.cpp
   ProtocolConformanceAnalysis.cpp
   RCIdentityAnalysis.cpp

lib/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.cpp

@@ -0,0 +1,30 @@
+//===--- NonLocalAccessBlockAnalysis.cpp  - Nonlocal end_access -----------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
+#include "swift/SIL/SILFunction.h"
+
+using namespace swift;
+
+// Populate this->accessBlocks with
+void NonLocalAccessBlocks::compute() {
+  for (SILBasicBlock &block : *this->function) {
+    for (SILInstruction &inst : block) {
+      if (auto *endAccess = dyn_cast<EndAccessInst>(&inst)) {
+        if (endAccess->getBeginAccess()->getParent() != endAccess->getParent())
+          this->accessBlocks.insert(&block);
+      } else if (isa<EndUnpairedAccessInst>(inst)) {
+        this->accessBlocks.insert(&block);
+      }
+    }
+  }
+}

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -52,6 +52,8 @@ class CopyPropagation : public SILFunctionTransform {
 /// Top-level pass driver.
 void CopyPropagation::run() {
   auto *f = getFunction();
+  auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
+  auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
 
   // Debug label for unit testing.
   LLVM_DEBUG(llvm::dbgs() << "*** CopyPropagation: " << f->getName() << "\n");
@@ -70,7 +72,8 @@ void CopyPropagation::run() {
     }
   }
   // Perform copy propgation for each copied value.
-  CanonicalizeOSSALifetime canonicalizer(pruneDebug);
+  CanonicalizeOSSALifetime canonicalizer(pruneDebug, accessBlockAnalysis,
+                                         dominanceAnalysis);
   for (auto &def : copiedDefs) {
     canonicalizer.canonicalizeValueLifetime(def);
     if (SILValue outerCopy = canonicalizer.createdOuterCopy()) {
@@ -81,7 +84,10 @@ void CopyPropagation::run() {
     // live ranges.
   }
   if (canonicalizer.hasChanged()) {
+    // Preserves NonLocalAccessBlockAnalysis.
+    accessBlockAnalysis->lockInvalidation();
     invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions);
+    accessBlockAnalysis->unlockInvalidation();
     DeadEndBlocks deBlocks(f);
     f->verifyOwnership(&deBlocks);
   }