Fix the mid-level pass pipeline

include/swift/SIL/ApplySite.h

@@ -583,6 +583,32 @@ class FullApplySite : public ApplySite {
 
 namespace llvm {
 
+template<>
+struct PointerLikeTypeTraits<swift::ApplySite> {
+public:
+  static inline void *getAsVoidPointer(swift::ApplySite apply) {
+    return (void*)apply.getInstruction();
+  }
+  static inline swift::ApplySite getFromVoidPointer(void *pointer) {
+    return swift::ApplySite((swift::SILInstruction*)pointer);
+  }
+  enum { NumLowBitsAvailable =
+         PointerLikeTypeTraits<swift::SILNode *>::NumLowBitsAvailable };
+};
+
+template<>
+struct PointerLikeTypeTraits<swift::FullApplySite> {
+public:
+  static inline void *getAsVoidPointer(swift::FullApplySite apply) {
+    return (void*)apply.getInstruction();
+  }
+  static inline swift::FullApplySite getFromVoidPointer(void *pointer) {
+    return swift::FullApplySite((swift::SILInstruction*)pointer);
+  }
+  enum { NumLowBitsAvailable =
+         PointerLikeTypeTraits<swift::SILNode *>::NumLowBitsAvailable };
+};
+
 // An ApplySite casts like a SILInstruction*.
 template <> struct simplify_type<const ::swift::ApplySite> {
   using SimpleType = ::swift::SILInstruction *;

include/swift/SILOptimizer/Analysis/ArraySemantic.h

@@ -45,6 +45,9 @@ enum class ArrayCallKind {
   kArrayUninitializedIntrinsic
 };
 
+/// Return true is the given function is an array semantics call.
+ArrayCallKind getArraySemanticsKind(SILFunction *f);
+
 /// Wrapper around array semantic calls.
 class ArraySemanticsCall {
   ApplyInst *SemanticsCall;
@@ -180,9 +183,6 @@ class ArraySemanticsCall {
 
   /// Could this array be backed by an NSArray.
   bool mayHaveBridgedObjectElementType() const;
-
-  /// Can this function be inlined by the early inliner.
-  bool canInlineEarly() const;
 
   /// If this is a call to  ArrayUninitialized (or
   /// ArrayUninitializedInstrinsic), identify the instructions that store

include/swift/SILOptimizer/PassManager/PassManager.h

@@ -258,16 +258,7 @@ class SILPassManager {
     }
   }
 
-  void executePassPipelinePlan(const SILPassPipelinePlan &Plan) {
-    for (const SILPassPipeline &Pipeline : Plan.getPipelines()) {
-      setStageName(Pipeline.Name);
-      resetAndRemoveTransformations();
-      for (PassKind Kind : Plan.getPipelinePasses(Pipeline)) {
-        addPass(Kind);
-      }
-      execute();
-    }
-  }
+  void executePassPipelinePlan(const SILPassPipelinePlan &Plan);
 
   void registerIRGenPass(PassKind Kind, SILTransform *Transform) {
     assert(IRGenPasses.find(unsigned(Kind)) == IRGenPasses.end() &&

include/swift/SILOptimizer/PassManager/PassPipeline.h

@@ -87,7 +87,7 @@ class SILPassPipelinePlan final {
 
   void print(llvm::raw_ostream &os);
 
-  void startPipeline(StringRef Name = "");
+  void startPipeline(StringRef Name = "", bool isFunctionPassPipeline = false);
   using PipelineKindIterator = decltype(Kinds)::const_iterator;
   using PipelineKindRange = iterator_range<PipelineKindIterator>;
   iterator_range<PipelineKindIterator>
@@ -104,11 +104,14 @@ struct SILPassPipeline final {
   unsigned ID;
   StringRef Name;
   unsigned KindOffset;
+  bool isFunctionPassPipeline;
 };
 
-inline void SILPassPipelinePlan::startPipeline(StringRef Name) {
+inline void SILPassPipelinePlan::
+startPipeline(StringRef Name, bool isFunctionPassPipeline) {
   PipelineStages.push_back(SILPassPipeline{
-      unsigned(PipelineStages.size()), Name, unsigned(Kinds.size())});
+      unsigned(PipelineStages.size()), Name, unsigned(Kinds.size()),
+      isFunctionPassPipeline});
 }
 
 inline SILPassPipelinePlan::PipelineKindRange

include/swift/SILOptimizer/Utils/Devirtualize.h

@@ -65,9 +65,11 @@ SubstitutionMap getWitnessMethodSubstitutions(SILModule &Module, ApplySite AI,
 ///
 /// If this succeeds, the caller must call deleteDevirtualizedApply on
 /// the original apply site.
-ApplySite tryDevirtualizeApply(ApplySite AI,
-                               ClassHierarchyAnalysis *CHA,
-                               OptRemark::Emitter *ORE = nullptr);
+///
+/// Return the new apply and true if the CFG was also modified.
+std::pair<ApplySite, bool>
+tryDevirtualizeApply(ApplySite AI, ClassHierarchyAnalysis *CHA,
+                     OptRemark::Emitter *ORE = nullptr);
 bool canDevirtualizeApply(FullApplySite AI, ClassHierarchyAnalysis *CHA);
 bool canDevirtualizeClassMethod(FullApplySite AI, ClassDecl *CD,
                                 OptRemark::Emitter *ORE = nullptr,
@@ -79,21 +81,23 @@ CanType getSelfInstanceType(CanType ClassOrMetatypeType);
 /// Devirtualize the given apply site, which is known to be devirtualizable.
 ///
 /// The caller must call deleteDevirtualizedApply on the original apply site.
-FullApplySite devirtualizeClassMethod(FullApplySite AI,
-                                      SILValue ClassInstance,
-                                      ClassDecl *CD,
-                                      OptRemark::Emitter *ORE);
+///
+/// Return the new apply and true if the CFG was also modified.
+std::pair<FullApplySite, bool> devirtualizeClassMethod(FullApplySite AI,
+                                                       SILValue ClassInstance,
+                                                       ClassDecl *CD,
+                                                       OptRemark::Emitter *ORE);
 
 /// Attempt to devirtualize the given apply site, which is known to be
 /// of a class method.  If this fails, the returned FullApplySite will be null.
 ///
 /// If this succeeds, the caller must call deleteDevirtualizedApply on
 /// the original apply site.
-FullApplySite
-tryDevirtualizeClassMethod(FullApplySite AI,
-                           SILValue ClassInstance,
-                           ClassDecl *CD,
-                           OptRemark::Emitter *ORE,
+///
+/// Return the new apply and true if the CFG was also modified.
+std::pair<FullApplySite, bool>
+tryDevirtualizeClassMethod(FullApplySite AI, SILValue ClassInstance,
+                           ClassDecl *CD, OptRemark::Emitter *ORE,
                            bool isEffectivelyFinalMethod = false);
 
 /// Attempt to devirtualize the given apply site, which is known to be
@@ -102,7 +106,9 @@ tryDevirtualizeClassMethod(FullApplySite AI,
 ///
 /// If this succeeds, the caller must call deleteDevirtualizedApply on
 /// the original apply site.
-ApplySite tryDevirtualizeWitnessMethod(ApplySite AI, OptRemark::Emitter *ORE);
+///
+/// Return the new apply and true if the CFG was also modified.
+std::pair<ApplySite, bool> tryDevirtualizeWitnessMethod(ApplySite AI, OptRemark::Emitter *ORE);
 
 /// Delete a successfully-devirtualized apply site.  This must always be
 /// called after devirtualizing an apply; not only is it not semantically

include/swift/SILOptimizer/Utils/PerformanceInlinerUtils.h

@@ -33,18 +33,25 @@ class SideEffectAnalysis;
 // Controls the decision to inline functions with @_semantics, @effect and
 // global_init attributes.
 enum class InlineSelection {
-  Everything,
-  NoGlobalInit, // and no availability semantics calls
-  NoSemanticsAndGlobalInit
+  PreModuleSerialization, // no @semantics, no @availability
+  RetainSemantics,        // Retain the lowest level of @semantic calls
+  Everything              // Full, including global init
 };
 
 // Returns the callee of an apply_inst if it is basically inlinable.
-SILFunction *getEligibleFunction(FullApplySite AI,
-                                 InlineSelection WhatToInline);
+SILFunction *
+getEligibleFunction(FullApplySite AI, InlineSelection WhatToInline,
+                    SmallPtrSetImpl<SILFunction *> &nestedSemanticFunctions);
 
 // Returns true if this is a pure call, i.e. the callee has no side-effects
 // and all arguments are constants.
 bool isPureCall(FullApplySite AI, SideEffectAnalysis *SEA);
+
+// Return true if the given function has a semantic annotation which may be
+// recognized by semantics passes. Such calls should only be inlined after all
+// semantic passes have been able to evaluate them.
+bool isOptimizableSemanticFunction(SILFunction *callee);
+
 } // end swift namespace
 
 //===----------------------------------------------------------------------===//

lib/SILOptimizer/Analysis/ArraySemantic.cpp

@@ -21,6 +21,44 @@
 
 using namespace swift;
 
+/// Determine which kind of array semantics function this is.
+ArrayCallKind swift::getArraySemanticsKind(SILFunction *f) {
+  ArrayCallKind Kind = ArrayCallKind::kNone;
+
+  for (auto &Attrs : f->getSemanticsAttrs()) {
+    auto Tmp =
+        llvm::StringSwitch<ArrayCallKind>(Attrs)
+            .Case("array.props.isNativeTypeChecked",
+                  ArrayCallKind::kArrayPropsIsNativeTypeChecked)
+            .StartsWith("array.init", ArrayCallKind::kArrayInit)
+            .Case("array.uninitialized", ArrayCallKind::kArrayUninitialized)
+            .Case("array.uninitialized_intrinsic", ArrayCallKind::kArrayUninitializedIntrinsic)
+            .Case("array.check_subscript", ArrayCallKind::kCheckSubscript)
+            .Case("array.check_index", ArrayCallKind::kCheckIndex)
+            .Case("array.get_count", ArrayCallKind::kGetCount)
+            .Case("array.get_capacity", ArrayCallKind::kGetCapacity)
+            .Case("array.get_element", ArrayCallKind::kGetElement)
+            .Case("array.make_mutable", ArrayCallKind::kMakeMutable)
+            .Case("array.get_element_address",
+                  ArrayCallKind::kGetElementAddress)
+            .Case("array.mutate_unknown", ArrayCallKind::kMutateUnknown)
+            .Case("array.reserve_capacity_for_append",
+                  ArrayCallKind::kReserveCapacityForAppend)
+            .Case("array.withUnsafeMutableBufferPointer",
+                  ArrayCallKind::kWithUnsafeMutableBufferPointer)
+            .Case("array.append_contentsOf", ArrayCallKind::kAppendContentsOf)
+            .Case("array.append_element", ArrayCallKind::kAppendElement)
+            .Default(ArrayCallKind::kNone);
+    if (Tmp != ArrayCallKind::kNone) {
+      assert(Kind == ArrayCallKind::kNone && "Multiple array semantic "
+                                             "strings?!");
+      Kind = Tmp;
+    }
+  }
+
+  return Kind;
+}
+
 static ParameterConvention
 getSelfParameterConvention(ApplyInst *SemanticsCall) {
   FunctionRefInst *FRI = cast<FunctionRefInst>(SemanticsCall->getCallee());
@@ -161,40 +199,7 @@ ArrayCallKind swift::ArraySemanticsCall::getKind() const {
   auto F = cast<FunctionRefInst>(SemanticsCall->getCallee())
                ->getInitiallyReferencedFunction();
 
-  ArrayCallKind Kind = ArrayCallKind::kNone;
-
-  for (auto &Attrs : F->getSemanticsAttrs()) {
-    auto Tmp =
-        llvm::StringSwitch<ArrayCallKind>(Attrs)
-            .Case("array.props.isNativeTypeChecked",
-                  ArrayCallKind::kArrayPropsIsNativeTypeChecked)
-            .StartsWith("array.init", ArrayCallKind::kArrayInit)
-            .Case("array.uninitialized", ArrayCallKind::kArrayUninitialized)
-            .Case("array.uninitialized_intrinsic", ArrayCallKind::kArrayUninitializedIntrinsic)
-            .Case("array.check_subscript", ArrayCallKind::kCheckSubscript)
-            .Case("array.check_index", ArrayCallKind::kCheckIndex)
-            .Case("array.get_count", ArrayCallKind::kGetCount)
-            .Case("array.get_capacity", ArrayCallKind::kGetCapacity)
-            .Case("array.get_element", ArrayCallKind::kGetElement)
-            .Case("array.make_mutable", ArrayCallKind::kMakeMutable)
-            .Case("array.get_element_address",
-                  ArrayCallKind::kGetElementAddress)
-            .Case("array.mutate_unknown", ArrayCallKind::kMutateUnknown)
-            .Case("array.reserve_capacity_for_append",
-                  ArrayCallKind::kReserveCapacityForAppend)
-            .Case("array.withUnsafeMutableBufferPointer",
-                  ArrayCallKind::kWithUnsafeMutableBufferPointer)
-            .Case("array.append_contentsOf", ArrayCallKind::kAppendContentsOf)
-            .Case("array.append_element", ArrayCallKind::kAppendElement)
-            .Default(ArrayCallKind::kNone);
-    if (Tmp != ArrayCallKind::kNone) {
-      assert(Kind == ArrayCallKind::kNone && "Multiple array semantic "
-                                             "strings?!");
-      Kind = Tmp;
-    }
-  }
-
-  return Kind;
+  return getArraySemanticsKind(F);
 }
 
 bool swift::ArraySemanticsCall::hasSelf() const {
@@ -585,26 +590,6 @@ bool swift::ArraySemanticsCall::mayHaveBridgedObjectElementType() const {
   return true;
 }
 
-bool swift::ArraySemanticsCall::canInlineEarly() const {
-  switch (getKind()) {
-    default:
-      return false;
-    case ArrayCallKind::kAppendContentsOf:
-    case ArrayCallKind::kReserveCapacityForAppend:
-    case ArrayCallKind::kAppendElement:
-    case ArrayCallKind::kArrayUninitializedIntrinsic:
-      // append(Element) calls other semantics functions. Therefore it's
-      // important that it's inlined by the early inliner (which is before all
-      // the array optimizations). Also, this semantics is only used to lookup
-      // Array.append(Element), so inlining it does not prevent any other
-      // optimization.
-      //
-      // Early inlining array.uninitialized_intrinsic semantic call helps in
-      // stack promotion.
-      return true;
-  }
-}
-
 SILValue swift::ArraySemanticsCall::getInitializationCount() const {
   if (getKind() == ArrayCallKind::kArrayUninitialized) {
     // Can be either a call to _adoptStorage or _allocateUninitialized.

lib/SILOptimizer/Analysis/EscapeAnalysis.cpp

@@ -1901,6 +1901,7 @@ void EscapeAnalysis::analyzeInstruction(SILInstruction *I,
       !isa<BeginApplyInst>(I)) {
     ArraySemanticsCall ASC(FAS.getInstruction());
     switch (ASC.getKind()) {
+      // TODO: Model ReserveCapacityForAppend, AppendContentsOf, AppendElement.
       case ArrayCallKind::kArrayPropsIsNativeTypeChecked:
       case ArrayCallKind::kCheckSubscript:
       case ArrayCallKind::kCheckIndex:

lib/SILOptimizer/FunctionSignatureTransforms/FunctionSignatureOpts.cpp

@@ -630,18 +630,6 @@ bool FunctionSignatureTransform::run(bool hasCaller) {
       TransformDescriptor.hasOnlyDirectInModuleCallers;
   SILFunction *F = TransformDescriptor.OriginalFunction;
 
-  // Never repeat the same function signature optimization on the same function.
-  // Multiple function signature optimizations are composed by successively
-  // optmizing the newly created functions. Each optimization creates a new
-  // level of thunk. Those should all be ultimately inlined away.
-  //
-  // This happens, for example, when a new reference to the original function is
-  // discovered during devirtualization. That will cause the original function
-  // (now and FSO thunk) to be pushed back on the function pass pipeline.
-  if (F->isThunk() == IsSignatureOptimizedThunk) {
-    LLVM_DEBUG(llvm::dbgs() << "  FSO already performed on this thunk\n");
-    return false;
-  }
 
   // If we are asked to assume a caller for testing purposes, set the flag.
   hasCaller |= FSOOptimizeIfNotCalled;
@@ -803,6 +791,19 @@ class FunctionSignatureOpts : public SILFunctionTransform {
       return;
     }
 
+    // Never repeat the same function signature optimization on the same
+    // function. Multiple function signature optimizations are composed by
+    // successively optmizing the newly created functions. Each optimization
+    // creates a new level of thunk which are all ultimately inlined away.
+    //
+    // This happens, for example, when a reference to the original function is
+    // discovered during devirtualization. That will cause the original function
+    // (now an FSO thunk) to be pushed back on the function pass pipeline.
+    if (F->isThunk() == IsSignatureOptimizedThunk) {
+      LLVM_DEBUG(llvm::dbgs() << "  FSO already performed on this thunk\n");
+      return;
+    }
+
     // Ok, we think we can perform optimization. Now perform a quick check
     auto *RCIA = getAnalysis<RCIdentityAnalysis>();
     auto *EA = PM->getAnalysis<EpilogueARCAnalysis>();

lib/SILOptimizer/LoopTransforms/LoopUnroll.cpp

@@ -192,7 +192,10 @@ static bool canAndShouldUnrollLoop(SILLoop *Loop, uint64_t TripCount) {
         ++Cost;
       if (auto AI = FullApplySite::isa(&Inst)) {
         auto Callee = AI.getCalleeFunction();
-        if (Callee && getEligibleFunction(AI, InlineSelection::Everything)) {
+        SmallPtrSet<SILFunction *, 1> nestedSemanticFunctions;
+        if (Callee
+            && getEligibleFunction(AI, InlineSelection::Everything,
+                                   nestedSemanticFunctions)) {
           // If callee is rather big and potentialy inlinable, it may be better
           // not to unroll, so that the body of the calle can be inlined later.
           Cost += Callee->size() * InsnsPerBB;

lib/SILOptimizer/Mandatory/MandatoryInlining.cpp

@@ -755,7 +755,7 @@ getCalleeFunction(SILFunction *F, FullApplySite AI, bool &IsThick,
 
 static SILInstruction *tryDevirtualizeApplyHelper(FullApplySite InnerAI,
                                                   ClassHierarchyAnalysis *CHA) {
-  auto NewInst = tryDevirtualizeApply(InnerAI, CHA);
+  auto NewInst = tryDevirtualizeApply(InnerAI, CHA).first;
   if (!NewInst)
     return InnerAI.getInstruction();
 

lib/SILOptimizer/PassManager/PassManager.cpp

@@ -592,6 +592,19 @@ void SILPassManager::runModulePass(unsigned TransIdx) {
   }
 }
 
+void SILPassManager::executePassPipelinePlan(const SILPassPipelinePlan &Plan) {
+  for (const SILPassPipeline &Pipeline : Plan.getPipelines()) {
+    setStageName(Pipeline.Name);
+    resetAndRemoveTransformations();
+    for (PassKind Kind : Plan.getPipelinePasses(Pipeline)) {
+      addPass(Kind);
+      assert(!Pipeline.isFunctionPassPipeline
+             || isa<SILFunctionTransform>(Transformations.back()));
+    }
+    execute();
+  }
+}
+
 void SILPassManager::execute() {
   const SILOptions &Options = getOptions();