SILVerifier - option to verify with or without linear lifetime check

Add a separate 'verifyOwnership()' entry point so it's possible
to check OSSA lifetimes at various points.

Move SILGenCleanup into a SILGen pass pipeline.

After SILGen, verify incomplete OSSA.

After SILGenCleanup, verify ownership.

Author: atrick

include/swift/SIL/SILFunction.h

@@ -1403,18 +1403,30 @@ class SILFunction
         decl->getLifetimeAnnotation());
   }
 
-  /// verify - Run the IR verifier to make sure that the SILFunction follows
+  /// verify - Run the SIL verifier to make sure that the SILFunction follows
   /// invariants.
-  void verify(bool SingleFunction = true) const;
+  void verify(bool SingleFunction = true,
+              bool isCompleteOSSA = true,
+              bool checkLinearLifetime = true) const;
+
+  /// Run the SIL verifier without assuming OSSA lifetimes end at dead end
+  /// blocks.
+  void verifyIncompleteOSSA() const {
+    verify(/*SingleFunction=*/true, /*completeOSSALifetimes=*/false);
+  }
 
   /// Verifies the lifetime of memory locations in the function.
   void verifyMemoryLifetime();
 
-  /// Run the SIL ownership verifier to check for ownership invariant failures.
+  /// Run the SIL ownership verifier to check that all values with ownership
+  /// have a linear lifetime. Regular OSSA invariants are checked separately in
+  /// normal SIL verification.
+  ///
+  /// \p deadEndBlocks is nullptr when OSSA lifetimes are complete.
   ///
-  /// NOTE: The ownership verifier is always run when performing normal IR
+  /// NOTE: The ownership verifier is run when performing normal IR
   /// verification, so this verification can be viewed as a subset of
-  /// SILFunction::verify.
+  /// SILFunction::verify(checkLinearLifetimes=true).
   void verifyOwnership(DeadEndBlocks *deadEndBlocks) const;
 
   /// Verify that all non-cond-br critical edges have been split.

include/swift/SIL/SILModule.h

@@ -904,7 +904,17 @@ class SILModule {
 
   /// Run the SIL verifier to make sure that all Functions follow
   /// invariants.
-  void verify() const;
+  void verify(bool isCompleteOSSA = true,
+              bool checkLinearLifetime = true) const;
+
+  /// Run the SIL verifier without assuming OSSA lifetimes end at dead end
+  /// blocks.
+  void verifyIncompleteOSSA() const {
+    verify(/*isCompleteOSSA=*/false);
+  }
+
+  /// Check linear OSSA lifetimes, assuming complete OSSA.
+  void verifyOwnership() const;
 
   /// Check if there are any leaking instructions.
   ///

include/swift/SIL/SILValue.h

@@ -675,6 +675,8 @@ class SILValue {
   }
 
   /// Verify that this SILValue and its uses respects ownership invariants.
+  ///
+  /// \p DEBlocks is nullptr when OSSA lifetimes are complete.
   void verifyOwnership(DeadEndBlocks *DEBlocks) const;
 
   SWIFT_DEBUG_DUMP;

include/swift/SILOptimizer/PassManager/PassPipeline.def

@@ -22,6 +22,7 @@
 #define PASSPIPELINE(NAME, DESCRIPTION)
 #endif
 
+PASSPIPELINE(SILGen, "SILGen Passes")
 PASSPIPELINE(Diagnostic, "Guaranteed Passes")
 PASSPIPELINE(LowerHopToActor, "Lower Hop to Actor")
 PASSPIPELINE(OwnershipEliminator, "Utility pass to just run the ownership eliminator pass")

lib/SIL/Verifier/LoadBorrowImmutabilityChecker.cpp

@@ -307,7 +307,7 @@ bool GatherWritesVisitor::visitUse(Operand *op, AccessUseType useTy) {
 //===----------------------------------------------------------------------===//
 
 LoadBorrowImmutabilityAnalysis::LoadBorrowImmutabilityAnalysis(
-    DeadEndBlocks &deadEndBlocks, const SILFunction *f)
+  DeadEndBlocks *deadEndBlocks, const SILFunction *f)
     : cache(GatherWrites(f)), deadEndBlocks(deadEndBlocks) {}
 
 // \p address may be an address, pointer, or box type.
@@ -333,7 +333,7 @@ bool LoadBorrowImmutabilityAnalysis::isImmutableInScope(
   for (auto *op : *writes) {
     auto *write = op->getUser();
     // First see if the write is a dead end block. In such a case, just skip it.
-    if (deadEndBlocks.isDeadEnd(write->getParent())) {
+    if (deadEndBlocks && deadEndBlocks->isDeadEnd(write->getParent())) {
       continue;
     }
 

lib/SIL/Verifier/SILOwnershipVerifier.cpp

@@ -86,7 +86,7 @@ class SILValueOwnershipChecker {
 
   /// A cache of dead-end basic blocks that we use to determine if we can
   /// ignore "leaks".
-  DeadEndBlocks &deadEndBlocks;
+  DeadEndBlocks *deadEndBlocks = nullptr;
 
   /// The value whose ownership we will check.
   SILValue value;
@@ -106,7 +106,8 @@ class SILValueOwnershipChecker {
   GuaranteedPhiVerifier &guaranteedPhiVerifier;
 
 public:
-  SILValueOwnershipChecker(DeadEndBlocks &deadEndBlocks, SILValue value,
+  /// \p deadEndBlocks is nullptr for complete OSSA lifetimes
+  SILValueOwnershipChecker(DeadEndBlocks *deadEndBlocks, SILValue value,
                            LinearLifetimeChecker::ErrorBuilder &errorBuilder,
                            GuaranteedPhiVerifier &guaranteedPhiVerifier)
       : result(), deadEndBlocks(deadEndBlocks), value(value),
@@ -485,7 +486,7 @@ bool SILValueOwnershipChecker::checkFunctionArgWithoutLifetimeEndingUses(
     break;
   }
 
-  if (deadEndBlocks.isDeadEnd(arg->getParent()))
+  if (deadEndBlocks && deadEndBlocks->isDeadEnd(arg->getParent()))
     return true;
 
   return !errorBuilder.handleMalformedSIL([&] {
@@ -503,9 +504,10 @@ bool SILValueOwnershipChecker::checkYieldWithoutLifetimeEndingUses(
   case OwnershipKind::None:
     return true;
   case OwnershipKind::Owned:
-    if (deadEndBlocks.isDeadEnd(yield->getParent()->getParent()))
+    if (deadEndBlocks
+        && deadEndBlocks->isDeadEnd(yield->getParent()->getParent())) {
       return true;
-
+    }
     return !errorBuilder.handleMalformedSIL([&] {
       llvm::errs() << "Owned yield without life ending uses!\n"
                    << "Value: " << *yield << '\n';
@@ -572,7 +574,7 @@ bool SILValueOwnershipChecker::checkValueWithoutLifetimeEndingUses(
     return true;
 
   if (auto *parentBlock = value->getParentBlock()) {
-    if (deadEndBlocks.isDeadEnd(parentBlock)) {
+    if (deadEndBlocks && deadEndBlocks->isDeadEnd(parentBlock)) {
       LLVM_DEBUG(llvm::dbgs() << "Ignoring transitively unreachable value "
                               << "without users!\n"
                               << "    Value: " << *value << '\n');
@@ -833,7 +835,7 @@ verifySILValueHelper(const SILFunction *f, SILValue value,
   if (!f->hasOwnership() || !f->shouldVerifyOwnership())
     return;
 
-  SILValueOwnershipChecker(*deadEndBlocks, value, errorBuilder,
+  SILValueOwnershipChecker(deadEndBlocks, value, errorBuilder,
                            guaranteedPhiVerifier)
       .check();
 }
@@ -882,11 +884,32 @@ void SILValue::verifyOwnership(DeadEndBlocks *deadEndBlocks) const {
   using BehaviorKind = LinearLifetimeChecker::ErrorBehaviorKind;
   LinearLifetimeChecker::ErrorBuilder errorBuilder(
       *f, BehaviorKind::PrintMessageAndAssert);
-  GuaranteedPhiVerifier guaranteedPhiVerifier(f, *deadEndBlocks, errorBuilder);
+  GuaranteedPhiVerifier guaranteedPhiVerifier(f, deadEndBlocks, errorBuilder);
   verifySILValueHelper(f, *this, errorBuilder, deadEndBlocks,
                        guaranteedPhiVerifier);
 }
 
+void SILModule::verifyOwnership() const {
+  if (DisableOwnershipVerification)
+    return;
+
+#ifdef NDEBUG
+  // When compiling without asserts enabled, only verify ownership if
+  // -sil-verify-all is set.
+  if (!getOptions().VerifyAll)
+    return;
+#endif
+
+  for (const SILFunction &function : *this) {
+    std::unique_ptr<DeadEndBlocks> deBlocks;
+    if (!getOptions().OSSACompleteLifetimes) {
+      deBlocks =
+        std::make_unique<DeadEndBlocks>(const_cast<SILFunction *>(&function));
+    }
+    function.verifyOwnership(deBlocks.get());
+  }
+}
+
 void SILFunction::verifyOwnership(DeadEndBlocks *deadEndBlocks) const {
   if (DisableOwnershipVerification)
     return;
@@ -919,7 +942,7 @@ void SILFunction::verifyOwnership(DeadEndBlocks *deadEndBlocks) const {
     errorBuilder.emplace(*this, BehaviorKind::PrintMessageAndAssert);
   }
 
-  GuaranteedPhiVerifier guaranteedPhiVerifier(this, *deadEndBlocks,
+  GuaranteedPhiVerifier guaranteedPhiVerifier(this, deadEndBlocks,
                                               *errorBuilder);
   for (auto &block : *this) {
     for (auto *arg : block.getArguments()) {

lib/SIL/Verifier/SILVerifier.cpp

@@ -702,17 +702,21 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
   const SILFunction &F;
   SILFunctionConventions fnConv;
   Lowering::TypeConverter &TC;
+
+  bool SingleFunction = true;
+  bool checkLinearLifetime = false;
+
   SmallVector<std::pair<StringRef, SILType>, 16> DebugVars;
   const SILInstruction *CurInstruction = nullptr;
   const SILArgument *CurArgument = nullptr;
   std::unique_ptr<DominanceInfo> Dominance;
 
   // Used for dominance checking within a basic block.
   llvm::DenseMap<const SILInstruction *, unsigned> InstNumbers;
-  
-  DeadEndBlocks DEBlocks;
+
+  std::unique_ptr<DeadEndBlocks> DEBlocks;
+
   LoadBorrowImmutabilityAnalysis loadBorrowImmutabilityAnalysis;
-  bool SingleFunction = true;
 
   /// A cache of the isOperandInValueUse check. When we process an operand, we
   /// fix this for each of its uses.
@@ -968,13 +972,13 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
   SILVerifier(const SILFunction &F, bool SingleFunction = true)
       : M(F.getModule().getSwiftModule()), F(F),
         fnConv(F.getConventionsInContext()), TC(F.getModule().Types),
+        SingleFunction(SingleFunction),
         Dominance(nullptr),
-        InstNumbers(numInstsInFunction(F)), DEBlocks(&F),
-        loadBorrowImmutabilityAnalysis(DEBlocks, &F),
-        SingleFunction(SingleFunction) {
+        InstNumbers(numInstsInFunction(F)),
+        loadBorrowImmutabilityAnalysis(DEBlocks.get(), &F) {
     if (F.isExternalDeclaration())
       return;
-      
+
     // Check to make sure that all blocks are well formed.  If not, the
     // SILVerifier object will explode trying to compute dominance info.
     unsigned InstIdx = 0;
@@ -1067,7 +1071,9 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
   void visitSILArgument(SILArgument *arg) {
     CurArgument = arg;
     checkLegalType(arg->getFunction(), arg, nullptr);
-    checkValueBaseOwnership(arg);
+    if (checkLinearLifetime) {
+      checkValueBaseOwnership(arg);
+    }
     if (auto *phiArg = dyn_cast<SILPhiArgument>(arg)) {
       if (phiArg->isPhi())
         visitSILPhiArgument(phiArg);
@@ -1107,7 +1113,9 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
 
     for (auto result : I->getResults()) {
       checkLegalType(F, result, I);
-      checkValueBaseOwnership(result);
+      if (checkLinearLifetime) {
+        checkValueBaseOwnership(result);
+      }
     }
   }
 
@@ -1125,7 +1133,7 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
               "Once ownership is gone, all values should have none ownership");
       return;
     }
-    SILValue(V).verifyOwnership(&DEBlocks);
+    SILValue(V).verifyOwnership(DEBlocks.get());
   }
 
   void checkSILInstruction(SILInstruction *I) {
@@ -6408,7 +6416,10 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
     }
   }
 
-  void verify() {
+  void verify(bool isCompleteOSSA, bool checkLinearLifetimes) {
+    if (!isCompleteOSSA || !F.getModule().getOptions().OSSACompleteLifetimes) {
+      DEBlocks = std::make_unique<DeadEndBlocks>(const_cast<SILFunction *>(&F));
+    }
     visitSILFunction(const_cast<SILFunction*>(&F));
   }
 };
@@ -6441,14 +6452,15 @@ static bool verificationEnabled(const SILModule &M) {
 
 /// verify - Run the SIL verifier to make sure that the SILFunction follows
 /// invariants.
-void SILFunction::verify(bool SingleFunction) const {
+void SILFunction::verify(bool SingleFunction, bool isCompleteOSSA,
+                         bool checkLinearLifetime) const {
   if (!verificationEnabled(getModule()))
     return;
 
   // Please put all checks in visitSILFunction in SILVerifier, not here. This
   // ensures that the pretty stack trace in the verifier is included with the
   // back trace when the verifier crashes.
-  SILVerifier(*this, SingleFunction).verify();
+  SILVerifier(*this, SingleFunction).verify(isCompleteOSSA, checkLinearLifetime);
 }
 
 void SILFunction::verifyCriticalEdges() const {
@@ -6717,7 +6729,7 @@ void SILGlobalVariable::verify() const {
 }
 
 /// Verify the module.
-void SILModule::verify() const {
+void SILModule::verify(bool isCompleteOSSA, bool checkLinearLifetime) const {
   if (!verificationEnabled(*this))
     return;
 
@@ -6732,7 +6744,7 @@ void SILModule::verify() const {
       llvm::errs() << "Symbol redefined: " << f.getName() << "!\n";
       assert(false && "triggering standard assertion failure routine");
     }
-    f.verify(/*singleFunction*/ false);
+    f.verify(/*singleFunction*/ false, isCompleteOSSA, checkLinearLifetime);
   }
 
   // Check all globals.

lib/SIL/Verifier/VerifierPrivate.h

@@ -28,10 +28,11 @@ namespace silverifier {
 
 class LoadBorrowImmutabilityAnalysis {
   SmallMultiMapCache<AccessPath, Operand *> cache;
-  DeadEndBlocks &deadEndBlocks;
+  DeadEndBlocks *deadEndBlocks = nullptr;
 
 public:
-  LoadBorrowImmutabilityAnalysis(DeadEndBlocks &deadEndBlocks,
+  /// \p deadEndBlocks should be nullptr to enforce complete OSSA lifetimes.
+  LoadBorrowImmutabilityAnalysis(DeadEndBlocks *deadEndBlocks,
                                  const SILFunction *f);
 
   /// Returns true if exhaustively lbi is guaranteed to never be invalidated by

lib/SILOptimizer/PassManager/PassPipeline.cpp

@@ -115,7 +115,6 @@ static void addDefiniteInitialization(SILPassPipelinePlan &P) {
 // should be in the -Onone pass pipeline and the prepare optimizations pipeline.
 static void addMandatoryDiagnosticOptPipeline(SILPassPipelinePlan &P) {
   P.startPipeline("Mandatory Diagnostic Passes + Enabling Optimization Passes");
-  P.addSILGenCleanup();
   P.addDiagnoseInvalidEscapingCaptures();
   P.addDiagnoseStaticExclusivity();
   P.addNestedSemanticFunctionCheck();
@@ -235,12 +234,21 @@ static void addMandatoryDiagnosticOptPipeline(SILPassPipelinePlan &P) {
 }
 
 SILPassPipelinePlan
-SILPassPipelinePlan::getDiagnosticPassPipeline(const SILOptions &Options) {
+SILPassPipelinePlan::getSILGenPassPipeline(const SILOptions &Options) {
   SILPassPipelinePlan P(Options);
+  P.startPipeline("SILGen Passes");
+
+  P.addSILGenCleanup();
 
   if (SILViewSILGenCFG) {
     addCFGPrinterPipeline(P, "SIL View SILGen CFG");
   }
+  return P;
+}
+
+SILPassPipelinePlan
+SILPassPipelinePlan::getDiagnosticPassPipeline(const SILOptions &Options) {
+  SILPassPipelinePlan P(Options);
 
   // If we are asked do debug serialization, instead of running all diagnostic
   // passes, just run mandatory inlining with dead transparent function cleanup

lib/SILOptimizer/PassManager/Passes.cpp

@@ -1,3 +1,4 @@
+
 //===--- Passes.cpp - Swift Compiler SIL Pass Entrypoints -----------------===//
 //
 // This source file is part of the Swift.org open source project
@@ -46,14 +47,22 @@ bool swift::runSILDiagnosticPasses(SILModule &Module) {
   auto &opts = Module.getOptions();
 
   // Verify the module, if required.
+  // OSSA lifetimes are incomplete until the SILGenCleanup pass runs.
   if (opts.VerifyAll)
-    Module.verify();
+    Module.verifyIncompleteOSSA();
 
   // If we parsed a .sil file that is already in canonical form, don't rerun
   // the diagnostic passes.
   if (Module.getStage() != SILStage::Raw)
     return false;
 
+  executePassPipelinePlan(&Module,
+                          SILPassPipelinePlan::getSILGenPassPipeline(opts),
+                          /*isMandatory*/ true);
+
+  if (opts.VerifyAll && opts.OSSACompleteLifetimes)
+    Module.verifyOwnership();
+
   executePassPipelinePlan(&Module,
                           SILPassPipelinePlan::getDiagnosticPassPipeline(opts),
                           /*isMandatory*/ true);