CanonicalOSSALifetime: Add support for overlapping access scopes.

include/swift/Basic/Compiler.h

@@ -121,11 +121,11 @@
 //   }
 // };
 #ifdef NDEBUG
-#define SWIFT_ASSERT_ONLY_DECL(X)
-#define SWIFT_ASSERT_ONLY(X) do { } while (false)
+#define SWIFT_ASSERT_ONLY_DECL(...)
+#define SWIFT_ASSERT_ONLY(...) do { } while (false)
 #else
-#define SWIFT_ASSERT_ONLY_DECL(X) X
-#define SWIFT_ASSERT_ONLY(X) do { X; } while (false)
+#define SWIFT_ASSERT_ONLY_DECL(X...) X
+#define SWIFT_ASSERT_ONLY(X...) do { X; } while (false)
 #endif
 
 #endif // SWIFT_BASIC_COMPILER_H

include/swift/SILOptimizer/Analysis/Analysis.def

@@ -38,6 +38,7 @@ ANALYSIS(Escape)
 ANALYSIS(InductionVariable)
 ANALYSIS(Loop)
 ANALYSIS(LoopRegion)
+ANALYSIS(NonLocalAccessBlock)
 ANALYSIS(OptimizerStats)
 ANALYSIS(PostDominance)
 ANALYSIS(PostOrder)

include/swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h

@@ -0,0 +1,97 @@
+//===--- NonLocalAccessBlockAnalysis.h - Nonlocal end_access ----*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// Cache the set of blocks that contain a non-local end_access, which is a rare
+/// occurrence. Optimizations that are driven by a known-use analysis, such as
+/// CanonicalOSSA, don't need to scan instructions that are unrelated to the SSA
+/// def-use graph. However, they may still need to be aware of unrelated access
+/// scope boundaries. By querying this analysis, they can avoid scanning all
+/// instructions just to deal with the extremely rare case of an end_access that
+/// spans blocks within the relevant SSA lifetime.
+///
+/// By default, this analysis is invalidated whenever instructions or blocks are
+/// changed, but it should ideally be preserved by passes that invalidate
+/// instructions but don't create any new access scopes or move end_access
+/// across blocks, which is unusual.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SILOPTIMIZER_ANALYSIS_NONLOCALACCESSBLOCKS_H
+#define SWIFT_SILOPTIMIZER_ANALYSIS_NONLOCALACCESSBLOCKS_H
+
+#include "swift/Basic/Compiler.h"
+#include "swift/SILOptimizer/Analysis/Analysis.h"
+#include "llvm/ADT/SmallPtrSet.h"
+
+namespace swift {
+
+class SILBasicBlock;
+class SILFunction;
+
+class NonLocalAccessBlocks {
+  friend class NonLocalAccessBlockAnalysis;
+
+  SILFunction *function;
+  llvm::SmallPtrSet<SILBasicBlock *, 4> accessBlocks;
+
+public:
+  NonLocalAccessBlocks(SILFunction *function) : function(function) {}
+
+  SILFunction *getFunction() const { return function; }
+
+  bool containsNonLocalEndAccess(SILBasicBlock *block) const {
+    return accessBlocks.count(block);
+  }
+
+  /// Perform NonLocalAccessBlockAnalysis for this function. Populate
+  /// this->accessBlocks with all blocks containing a non-local end_access.
+  void compute();
+};
+
+class NonLocalAccessBlockAnalysis
+    : public FunctionAnalysisBase<NonLocalAccessBlocks> {
+public:
+  static bool classof(const SILAnalysis *S) {
+    return S->getKind() == SILAnalysisKind::NonLocalAccessBlock;
+  }
+  NonLocalAccessBlockAnalysis()
+      : FunctionAnalysisBase<NonLocalAccessBlocks>(
+          SILAnalysisKind::NonLocalAccessBlock) {}
+
+  NonLocalAccessBlockAnalysis(const NonLocalAccessBlockAnalysis &) = delete;
+
+  NonLocalAccessBlockAnalysis &
+  operator=(const NonLocalAccessBlockAnalysis &) = delete;
+
+protected:
+  virtual std::unique_ptr<NonLocalAccessBlocks>
+  newFunctionAnalysis(SILFunction *function) override {
+    auto result = std::make_unique<NonLocalAccessBlocks>(function);
+    result->compute();
+    return result;
+  }
+
+  virtual bool shouldInvalidate(SILAnalysis::InvalidationKind kind) override {
+    return kind & InvalidationKind::BranchesAndInstructions;
+  }
+
+  SWIFT_ASSERT_ONLY_DECL(
+      virtual void verify(NonLocalAccessBlocks *accessBlocks) const override {
+        NonLocalAccessBlocks checkAccessBlocks(accessBlocks->function);
+        checkAccessBlocks.compute();
+        assert(checkAccessBlocks.accessBlocks == accessBlocks->accessBlocks);
+      })
+};
+
+} // end namespace swift
+
+#endif

include/swift/SILOptimizer/Utils/CanonicalOSSALifetime.h

@@ -93,10 +93,12 @@
 #ifndef SWIFT_SILOPTIMIZER_UTILS_CANONICALOSSALIFETIME_H
 #define SWIFT_SILOPTIMIZER_UTILS_CANONICALOSSALIFETIME_H
 
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SetVector.h"
 #include "swift/SIL/SILInstruction.h"
+#include "swift/SILOptimizer/Analysis/DominanceAnalysis.h"
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Utils/PrunedLiveness.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SetVector.h"
 
 namespace swift {
 
@@ -187,6 +189,13 @@ class CanonicalizeOSSALifetime {
   /// liveness may be pruned during canonicalization.
   bool pruneDebug;
 
+  NonLocalAccessBlockAnalysis *accessBlockAnalysis;
+  // Lazilly initialize accessBlocks only when
+  // extendLivenessThroughOverlappingAccess is invoked.
+  NonLocalAccessBlocks *accessBlocks = nullptr;
+
+  DominanceAnalysis *dominanceAnalysis;
+
   /// Current copied def for which this state describes the liveness.
   SILValue currentDef;
 
@@ -208,7 +217,7 @@ class CanonicalizeOSSALifetime {
   /// Record all interesting debug_value instructions here rather then treating
   /// them like a normal use. An interesting debug_value is one that may lie
   /// outisde the pruned liveness at the time it is discovered.
-  llvm::SmallDenseSet<DebugValueInst *, 8> debugValues;
+  llvm::SmallPtrSet<DebugValueInst *, 8> debugValues;
 
   /// Reuse a general worklist for def-use traversal.
   SmallSetVector<SILValue, 8> defUseWorklist;
@@ -226,12 +235,19 @@ class CanonicalizeOSSALifetime {
   CanonicalOSSAConsumeInfo consumes;
 
 public:
-  CanonicalizeOSSALifetime(bool pruneDebug) : pruneDebug(pruneDebug) {}
+  CanonicalizeOSSALifetime(bool pruneDebug,
+                           NonLocalAccessBlockAnalysis *accessBlockAnalysis,
+                           DominanceAnalysis *dominanceAnalysis)
+    : pruneDebug(pruneDebug), accessBlockAnalysis(accessBlockAnalysis),
+      dominanceAnalysis(dominanceAnalysis) {}
 
   SILValue getCurrentDef() const { return currentDef; }
 
   void initDef(SILValue def) {
     assert(consumingBlocks.empty() && debugValues.empty() && liveness.empty());
+    // Clear the cached analysis pointer just in case the client invalidates the
+    // analysis, freeing its memory.
+    accessBlocks = nullptr;
     consumes.clear();
 
     currentDef = def;
@@ -281,6 +297,10 @@ class CanonicalizeOSSALifetime {
 
   bool computeCanonicalLiveness();
 
+  bool endsAccessOverlappingPrunedBoundary(SILInstruction *inst);
+
+  void extendLivenessThroughOverlappingAccess();
+
   void findOrInsertDestroyInBlock(SILBasicBlock *bb);
 
   void findOrInsertDestroys();

include/swift/SILOptimizer/Utils/PrunedLiveness.h

@@ -147,7 +147,7 @@ class PrunedLiveBlocks {
   }
 
   /// Update this liveness result for a single use.
-  IsLive updateForUse(Operand *use);
+  IsLive updateForUse(SILInstruction *user);
 
   IsLive getBlockLiveness(SILBasicBlock *bb) const {
     auto liveBlockIter = liveBlocks.find(bb);
@@ -217,7 +217,7 @@ class PrunedLiveness {
   /// relationships that generate liveness. For example, use->isLifetimeEnding()
   /// cannot distinguish the end of the borrow scope that defines this extended
   /// live range vs. a nested borrow scope within the extended live range.
-  void updateForUse(Operand *use, bool lifetimeEnding);
+  void updateForUse(SILInstruction *user, bool lifetimeEnding);
 
   PrunedLiveBlocks::IsLive getBlockLiveness(SILBasicBlock *bb) const {
     return liveBlocks.getBlockLiveness(bb);

lib/SILOptimizer/Analysis/Analysis.cpp

@@ -20,6 +20,7 @@
 #include "swift/SILOptimizer/Analysis/ClassHierarchyAnalysis.h"
 #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h"
 #include "swift/SILOptimizer/Analysis/IVAnalysis.h"
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Analysis/PostOrderAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ProtocolConformanceAnalysis.h"
 #include "swift/SILOptimizer/Utils/InstOptUtils.h"
@@ -60,3 +61,7 @@ SILAnalysis *swift::createBasicCalleeAnalysis(SILModule *M) {
 SILAnalysis *swift::createProtocolConformanceAnalysis(SILModule *M) {
   return new ProtocolConformanceAnalysis(M);
 }
+
+SILAnalysis *swift::createNonLocalAccessBlockAnalysis(SILModule *M) {
+  return new NonLocalAccessBlockAnalysis();
+}

lib/SILOptimizer/Analysis/CMakeLists.txt

@@ -19,6 +19,7 @@ target_sources(swiftSILOptimizer PRIVATE
   LoopAnalysis.cpp
   LoopRegionAnalysis.cpp
   MemoryBehavior.cpp
+  NonLocalAccessBlockAnalysis.cpp
   PassManagerVerifierAnalysis.cpp
   ProtocolConformanceAnalysis.cpp
   RCIdentityAnalysis.cpp

lib/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.cpp

@@ -0,0 +1,30 @@
+//===--- NonLocalAccessBlockAnalysis.cpp  - Nonlocal end_access -----------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
+#include "swift/SIL/SILFunction.h"
+
+using namespace swift;
+
+// Populate this->accessBlocks with
+void NonLocalAccessBlocks::compute() {
+  for (SILBasicBlock &block : *this->function) {
+    for (SILInstruction &inst : block) {
+      if (auto *endAccess = dyn_cast<EndAccessInst>(&inst)) {
+        if (endAccess->getBeginAccess()->getParent() != endAccess->getParent())
+          this->accessBlocks.insert(&block);
+      } else if (isa<EndUnpairedAccessInst>(inst)) {
+        this->accessBlocks.insert(&block);
+      }
+    }
+  }
+}

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -52,6 +52,8 @@ class CopyPropagation : public SILFunctionTransform {
 /// Top-level pass driver.
 void CopyPropagation::run() {
   auto *f = getFunction();
+  auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
+  auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
 
   // Debug label for unit testing.
   LLVM_DEBUG(llvm::dbgs() << "*** CopyPropagation: " << f->getName() << "\n");
@@ -70,7 +72,8 @@ void CopyPropagation::run() {
     }
   }
   // Perform copy propgation for each copied value.
-  CanonicalizeOSSALifetime canonicalizer(pruneDebug);
+  CanonicalizeOSSALifetime canonicalizer(pruneDebug, accessBlockAnalysis,
+                                         dominanceAnalysis);
   for (auto &def : copiedDefs) {
     canonicalizer.canonicalizeValueLifetime(def);
     if (SILValue outerCopy = canonicalizer.createdOuterCopy()) {
@@ -81,7 +84,10 @@ void CopyPropagation::run() {
     // live ranges.
   }
   if (canonicalizer.hasChanged()) {
+    // Preserves NonLocalAccessBlockAnalysis.
+    accessBlockAnalysis->lockInvalidation();
     invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions);
+    accessBlockAnalysis->unlockInvalidation();
     DeadEndBlocks deBlocks(f);
     f->verifyOwnership(&deBlocks);
   }