Improve recursion-breaking when resolving type witnesses.

Specifically track when we're resolving type witnesses for a given
conformance, and refuse to recursively attempt to resolve those type
witnesses. The standard library patch in rdar://problem/21789238
triggers this, and isolating it in a small test case has proven
illusive.

Swift SVN r30160

Author: DougGregor

include/swift/AST/ProtocolConformance.h

@@ -69,17 +69,16 @@ enum class ProtocolConformanceKind {
 };
 
 /// Describes the state of a protocol conformance, which may be complete,
-/// incomplete, or invalid.
+/// incomplete, or currently being checked.
 enum class ProtocolConformanceState {
-  /// The conformance has been fully checked and is complete and well-formed.
+  /// The conformance has been fully checked.
   Complete,
   /// The conformance is known but is not yet complete.
   Incomplete,
+  /// The conformance's type witnesses are currently being resolved.
+  CheckingTypeWitnesses,
   /// The conformance is being checked.
   Checking,
-  /// The conformance has been found to be invalid and should not be
-  /// used.
-  Invalid
 };
 
 /// \brief Describes how a particular type conforms to a given protocol,
@@ -119,19 +118,18 @@ class ProtocolConformance {
   /// Retrieve the state of this conformance.
   ProtocolConformanceState getState() const;
 
-  /// Determine whether this conformance is complete and well-formed.
+  /// Determine whether this conformance is complete.
   bool isComplete() const {
     return getState() == ProtocolConformanceState::Complete;
   }
 
   /// Determine whether this conformance is invalid.
-  bool isInvalid() const {
-    return getState() == ProtocolConformanceState::Invalid;
-  }
+  bool isInvalid() const;
 
   /// Determine whether this conformance is incomplete.
   bool isIncomplete() const {
     return getState() == ProtocolConformanceState::Incomplete ||
+           getState() == ProtocolConformanceState::CheckingTypeWitnesses ||
            getState() == ProtocolConformanceState::Checking;
   }
 
@@ -295,7 +293,9 @@ class NormalProtocolConformance : public ProtocolConformance,
 
   /// The declaration context containing the ExtensionDecl or
   /// NominalTypeDecl that declared the conformance.
-  DeclContext *DC;
+  ///
+  /// Also stores the "invalid" bit.
+  llvm::PointerIntPair<DeclContext *, 1, bool> DCAndInvalid;
 
   /// \brief The mapping of individual requirements in the protocol over to
   /// the declarations that satisfy those requirements.
@@ -319,7 +319,7 @@ class NormalProtocolConformance : public ProtocolConformance,
                             SourceLoc loc, DeclContext *dc,
                             ProtocolConformanceState state)
     : ProtocolConformance(ProtocolConformanceKind::Normal, conformingType),
-      ProtocolAndState(protocol, state), Loc(loc), DC(dc)
+      ProtocolAndState(protocol, state), Loc(loc), DCAndInvalid(dc, false)
   {
   }
 
@@ -332,7 +332,7 @@ class NormalProtocolConformance : public ProtocolConformance,
 
   /// Get the declaration context that contains the conforming extension or
   /// nominal type declaration.
-  DeclContext *getDeclContext() const { return DC; }
+  DeclContext *getDeclContext() const { return DCAndInvalid.getPointer(); }
 
   /// Retrieve the state of this conformance.
   ProtocolConformanceState getState() const {
@@ -344,6 +344,12 @@ class NormalProtocolConformance : public ProtocolConformance,
     ProtocolAndState.setInt(state);
   }
 
+  /// Determine whether this conformance is invalid.
+  bool isInvalid() const { return DCAndInvalid.getInt(); }
+
+  /// Mark this conformance as invalid.
+  void setInvalid() { DCAndInvalid.setInt(true); }
+
   /// Retrieve the type witness substitution and type decl (if one exists)
   /// for the given associated type.
   std::pair<const Substitution &, TypeDecl *>
@@ -628,6 +634,10 @@ class InheritedProtocolConformance : public ProtocolConformance,
   }
 };
 
+inline bool ProtocolConformance::isInvalid() const {
+  return getRootNormalConformance()->isInvalid();
+}
+
 } // end namespace swift
 
 #endif // LLVM_SWIFT_AST_PROTOCOLCONFORMANCE_H

lib/AST/Module.cpp

@@ -731,6 +731,11 @@ ArrayRef<Substitution> BoundGenericType::getSubstitutions(
     ++index;
   }
 
+  // Before recording substitutions, make sure we didn't end up doing it
+  // recursively.
+  if (auto known = ctx.getSubstitutions(canon, gpContext))
+    return *known;
+
   // Copy and record the substitutions.
   auto permanentSubs = ctx.AllocateCopy(resultSubstitutions,
                                         hasTypeVariables

lib/AST/ProtocolConformance.cpp

@@ -233,7 +233,8 @@ ConcreteDeclRef NormalProtocolConformance::getWitness(
   if (known != Mapping.end()) {
     return known->second;
   } else {
-    assert(!isComplete() && "Resolver did not resolve requirement");
+    assert((!isComplete() || isInvalid()) &&
+           "Resolver did not resolve requirement");
     return ConcreteDeclRef();
   }
 }

lib/AST/Type.cpp

@@ -2497,11 +2497,13 @@ static Type getMemberForBaseType(Module *module,
       return Type();
 
     case ConformanceKind::Conforms:
-      // If we're supposed to skip this conformance's unsatisfied type
+      // If we have an unsatisfied type witness while we're checking the
+      // conformances we're supposed to skip this conformance's unsatisfied type
       // witnesses, and we have an unsatisfied type witness, return
       // "missing".
-      if (conformance.getPointer()->getRootNormalConformance()
-            == options.getSkippedConformance() &&
+      if (conformance.getPointer() &&
+          conformance.getPointer()->getRootNormalConformance()->getState()
+            == ProtocolConformanceState::CheckingTypeWitnesses &&
           !conformance.getPointer()->hasTypeWitness(assocType, nullptr))
         return Type();
 

lib/AST/Verifier.cpp

@@ -1661,14 +1661,14 @@ struct ASTNodeBase {};
 
       switch (conformance->getState()) {
       case ProtocolConformanceState::Complete:
-      case ProtocolConformanceState::Invalid:
         // More checking below.
         break;
 
       case ProtocolConformanceState::Incomplete:
         // Ignore incomplete conformances; we didn't need them.
         return;
 
+      case ProtocolConformanceState::CheckingTypeWitnesses:
       case ProtocolConformanceState::Checking:
         dumpRef(decl);
         Out << " has a protocol conformance that is still being checked "