[sil-generic-specializer] Avoid unlimited generic specialization of very deeply nested bound generic types

lib/SILOptimizer/Utils/Generics.cpp

@@ -19,6 +19,29 @@
 
 using namespace swift;
 
+// Max depth of a bound generic which can be processed by the generic
+// specializer.
+// E.g. the depth of Array<Array<Array<T>>> is 3.
+// No specializations will be produced, if any of generic parameters contains
+// a bound generic type with the depth higher than this threshold 
+static const unsigned BoundGenericDepthThreshold = 50;
+
+static unsigned getBoundGenericDepth(Type t) {
+  unsigned Depth = 0;
+  if (auto BGT = t->getAs<BoundGenericType>()) {
+    Depth++;
+    auto GenericArgs = BGT->getGenericArgs();
+    unsigned MaxGenericArgDepth = 0;
+    for (auto GenericArg : GenericArgs) {
+      auto ArgDepth = getBoundGenericDepth(GenericArg);
+      if (ArgDepth > MaxGenericArgDepth)
+        MaxGenericArgDepth = ArgDepth;
+    }
+    Depth += MaxGenericArgDepth;
+  }
+  return Depth;
+}
+
 // =============================================================================
 // ReabstractionInfo
 // =============================================================================
@@ -50,6 +73,19 @@ ReabstractionInfo::ReabstractionInfo(SILFunction *OrigF,
     DEBUG(llvm::dbgs() << "    Cannot specialize with dynamic self.\n");
     return;
   }
+
+  // Check if the substitution contains any generic types that are too deep.
+  // If this is the case, bail to avoid the explosion in the number of 
+  // generated specializations.
+  for (auto Sub : ParamSubs) {
+    auto Replacement = Sub.getReplacement();
+    if (Replacement.findIf([](Type ty) -> bool {
+          return getBoundGenericDepth(ty) >= BoundGenericDepthThreshold;
+        })) {
+      return;
+    }
+  }
+
   SILModule &M = OrigF->getModule();
   Module *SM = M.getSwiftModule();
 

test/SILOptimizer/specialize_deep_generics.swift

@@ -0,0 +1,34 @@
+// RUN: %target-swift-frontend -O -emit-sil %s | FileCheck %s
+
+// Check that this Church Numerals inspired example does not hang
+// a compiler in the generic specializer.
+//
+// rdar://problem/21260480
+
+protocol Nat {
+  init()
+  // static stored properties are not supported in generic structs.
+  var val : Int32 {get}
+}
+
+struct Zero : Nat { var val : Int32 = 0 }
+
+struct PlusOne<X : Nat> : Nat {
+  var val : Int32 = X().val + 1
+}
+
+// Compiler used to keep performing the generic specialization of
+// computeNat for increasingly deeply nested bound generic types
+// like  PlusOne<PlusOne<....<PlusOne<Zero>>>
+func computeNat<T: Nat>(_ v : Int32, _ t: T) -> Int32 {
+  if v == 0 {
+    return t.val
+  }
+  return computeNat(v - 1, PlusOne<T>())
+}
+
+// CHECK-LABEL: sil @_TF24specialize_deep_generics14testComputeNatFT_Vs5Int32
+public func testComputeNat() -> Int32 {
+ return computeNat(8, Zero())
+}
+