RequirementMachine: Skip protocol type aliases that contain unbound dependent member types

lib/AST/RequirementMachine/HomotopyReduction.cpp

@@ -611,10 +611,11 @@ GenericSignatureErrors RewriteSystem::getErrors() const {
     if (!isInMinimizationDomain(rule.getLHS().getRootProtocol()))
       continue;
 
-    if (!rule.isRedundant() &&
-        !rule.isProtocolTypeAliasRule() &&
-        rule.containsNameSymbols())
-      result |= GenericSignatureErrorFlags::HasInvalidRequirements;
+    if (!rule.isRedundant()) {
+      if (!rule.isProtocolTypeAliasRule() &&
+          rule.containsNameSymbols())
+        result |= GenericSignatureErrorFlags::HasInvalidRequirements;
+    }
 
     if (rule.isRecursive())
       result |= GenericSignatureErrorFlags::HasInvalidRequirements;
@@ -624,12 +625,9 @@ GenericSignatureErrors RewriteSystem::getErrors() const {
         if (property->getKind() == Symbol::Kind::ConcreteConformance)
           result |= GenericSignatureErrorFlags::HasConcreteConformances;
 
-        if (property->hasSubstitutions()) {
-          for (auto t : property->getSubstitutions()) {
-            if (t.containsNameSymbols())
-              result |= GenericSignatureErrorFlags::HasInvalidRequirements;
-          }
-        }
+        if (property->hasSubstitutions() &&
+            property->containsNameSymbols())
+          result |= GenericSignatureErrorFlags::HasInvalidRequirements;
       }
     }
   }
@@ -661,10 +659,20 @@ RewriteSystem::getMinimizedProtocolRules() const {
     if (!isInMinimizationDomain(proto))
       continue;
 
-    if (rule.isProtocolTypeAliasRule())
+    if (rule.isProtocolTypeAliasRule()) {
+      if (auto property = rule.isPropertyRule()) {
+        if (property->containsNameSymbols())
+          continue;
+      } else if (rule.getRHS().containsNameSymbols()) {
+        continue;
+      }
       rules[proto].TypeAliases.push_back(ruleID);
-    else if (!rule.containsNameSymbols())
+    } else {
+      if (rule.containsNameSymbols())
+        continue;
+
       rules[proto].Requirements.push_back(ruleID);
+    }
   }
 
   return rules;

lib/AST/RequirementMachine/Symbol.cpp

@@ -686,6 +686,15 @@ Symbol Symbol::transformConcreteSubstitutions(
   return withConcreteSubstitutions(substitutions, ctx);
 }
 
+bool Symbol::containsNameSymbols() const {
+  for (auto t : getSubstitutions()) {
+    if (t.containsNameSymbols())
+      return true;
+  }
+
+  return false;
+}
+
 /// Print the symbol using our mnemonic representation.
 void Symbol::dump(llvm::raw_ostream &out) const {
   llvm::DenseMap<CanType, Identifier> substitutionNames;

lib/AST/RequirementMachine/Symbol.h

@@ -242,6 +242,8 @@ class Symbol final {
       const MutableTerm &prefix,
       RewriteContext &ctx) const;
 
+  bool containsNameSymbols() const;
+
   void dump(llvm::raw_ostream &out) const;
 
   friend bool operator==(Symbol lhs, Symbol rhs) {

lib/Serialization/Serialization.cpp

@@ -1727,7 +1727,10 @@ void Serializer::writeRequirementSignature(
 
   for (const auto &typeAlias : requirementSig.getTypeAliases()) {
     rawData.push_back(addDeclBaseNameRef(typeAlias.getName()));
-    rawData.push_back(addTypeRef(typeAlias.getUnderlyingType()));
+
+    auto underlyingType = typeAlias.getUnderlyingType();
+    ASSERT(!underlyingType->findUnresolvedDependentMemberType());
+    rawData.push_back(addTypeRef(underlyingType));
   }
 
   RequirementSignatureLayout::emitRecord(

test/Generics/rdar136686001.swift

@@ -0,0 +1,23 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-swift-frontend -emit-module %s -emit-module-path %t/out.swiftmodule
+// RUN: %target-swift-frontend -typecheck %s -debug-generic-signatures 2>&1 | %FileCheck %s
+
+public struct S {
+  public typealias A = Int
+}
+
+public protocol P {
+  typealias A = S
+}
+
+public struct G<T> {}
+
+public protocol Q: P {
+  typealias B = G<Self.A.A>
+}
+
+// FIXME: This should be diagnosed as an error.
+
+// CHECK-LABEL: rdar136686001.(file).f@
+// CHECK-NEXT: Generic signature: <T, U where T : Q>
+public func f<T: Q, U>(_: T, _: U) where T.B == U {}