WIP: Closure ABI - Make @noescape Swift closures trivial

include/swift/AST/Types.h

@@ -3843,6 +3843,12 @@ class SILFunctionType final : public TypeBase, public llvm::FoldingSetNode,
     return getExtInfo().isNoEscape();
   }
 
+  /// Thick swift noescape function types are trivial.
+  bool isTrivialNoEscape() const {
+    return isNoEscape() &&
+           getRepresentation() == SILFunctionTypeRepresentation::Thick;
+  }
+
   bool isNoReturnFunction(); // Defined in SILType.cpp
 
   class ABICompatibilityCheckResult {
@@ -3851,6 +3857,7 @@ class SILFunctionType final : public TypeBase, public llvm::FoldingSetNode,
     enum innerty {
       None,
       DifferentFunctionRepresentations,
+      ABIEscapeToNoEscapeConversion,
       DifferentNumberOfResults,
       DifferentReturnValueConventions,
       ABIIncompatibleReturnValues,
@@ -3870,6 +3877,10 @@ class SILFunctionType final : public TypeBase, public llvm::FoldingSetNode,
     ABICompatibilityCheckResult() = delete;
 
     bool isCompatible() const { return kind == innerty::None; }
+    bool isCompatibleUpToNoEscapeConversion() {
+      return kind == innerty::None ||
+             kind == innerty::ABIEscapeToNoEscapeConversion;
+    }
 
     bool hasPayload() const { return payload.hasValue(); }
     uintptr_t getPayload() const { return payload.getValue(); }

include/swift/SIL/InstructionUtils.h

@@ -102,7 +102,8 @@ bool isIncidentalUse(SILInstruction *user);
 /// only used in recognizable patterns without otherwise "escaping".
 bool onlyAffectsRefCount(SILInstruction *user);
 
-/// If V is a convert_function, return its operand recursively.
+/// If V is a convert_function or convert_escape_to_noescape return its operand
+/// recursively.
 SILValue stripConvertFunctions(SILValue V);
 
 /// Given an address accessed by an instruction that reads or modifies

include/swift/SIL/SILInstruction.h

@@ -3830,7 +3830,13 @@ class ConvertFunctionInst final
   ConvertFunctionInst(SILDebugLocation DebugLoc, SILValue Operand,
                       ArrayRef<SILValue> TypeDependentOperands, SILType Ty)
       : UnaryInstructionWithTypeDependentOperandsBase(
-            DebugLoc, Operand, TypeDependentOperands, Ty) {}
+            DebugLoc, Operand, TypeDependentOperands, Ty) {
+    assert((Operand->getType().castTo<SILFunctionType>()->isNoEscape() ==
+                Ty.castTo<SILFunctionType>()->isNoEscape() ||
+            Ty.castTo<SILFunctionType>()->getRepresentation() !=
+                SILFunctionType::Representation::Thick) &&
+           "Change of escapeness is not ABI compatible");
+  }
 
   static ConvertFunctionInst *
   create(SILDebugLocation DebugLoc, SILValue Operand, SILType Ty,

include/swift/SILOptimizer/Analysis/CFG.h

@@ -32,6 +32,7 @@ class SILBasicBlock;
 ///
 /// 1. Has a return terminator.
 /// 2. unreachable + noreturn terminator sequence.
+/// 3. has a throw terminator.
 ///
 /// If we just have an unreachable without a noreturn call before it, we must
 /// have a failure BB.

lib/IRGen/GenCall.cpp

@@ -1919,6 +1919,8 @@ void CallEmission::setFromCallee() {
 
     // Fill in the context pointer if necessary.
     if (!contextPtr) {
+      assert(!CurCallee.getOrigFunctionType()->isNoEscape() &&
+             "Missing context?");
       contextPtr = llvm::UndefValue::get(IGF.IGM.RefCountedPtrTy);
     }
   }
@@ -3369,14 +3371,17 @@ Callee irgen::getBlockPointerCallee(IRGenFunction &IGF,
   return Callee(std::move(info), fn, blockPtr);
 }
 
-Callee irgen::getSwiftFunctionPointerCallee(IRGenFunction &IGF,
-                                            llvm::Value *fnPtr,
-                                            llvm::Value *dataPtr,
-                                            CalleeInfo &&calleeInfo) {
+Callee irgen::getSwiftFunctionPointerCallee(
+    IRGenFunction &IGF, llvm::Value *fnPtr, llvm::Value *dataPtr,
+    CalleeInfo &&calleeInfo, bool castOpaqueToRefcountedContext) {
   auto sig = emitCastOfFunctionPointer(IGF, fnPtr, calleeInfo.OrigFnType);
 
   FunctionPointer fn(fnPtr, sig);
-
+  if (castOpaqueToRefcountedContext) {
+    assert(dataPtr && dataPtr->getType() == IGF.IGM.OpaquePtrTy &&
+           "Expecting trivial closure context");
+    dataPtr = IGF.Builder.CreateBitCast(dataPtr, IGF.IGM.RefCountedPtrTy);
+  }
   return Callee(std::move(calleeInfo), fn, dataPtr);
 }
 

lib/IRGen/GenCall.h

@@ -120,7 +120,8 @@ namespace irgen {
   Callee getSwiftFunctionPointerCallee(IRGenFunction &IGF,
                                        llvm::Value *fnPtr,
                                        llvm::Value *contextPtr,
-                                       CalleeInfo &&info);
+                                       CalleeInfo &&info,
+                                       bool castOpaqueToRefcountedContext);
 
   Address emitAllocYieldOnceCoroutineBuffer(IRGenFunction &IGF);
   void emitDeallocYieldOnceCoroutineBuffer(IRGenFunction &IGF, Address buffer);

lib/IRGen/GenFunc.cpp

@@ -578,12 +578,16 @@ const TypeInfo *TypeConverter::convertFunctionType(SILFunctionType *T) {
     spareBits.append(IGM.getFunctionPointerSpareBits());
     spareBits.append(IGM.getHeapObjectSpareBits());
 
-    return FuncTypeInfo::create(CanSILFunctionType(T),
-                                IGM.FunctionPairTy,
-                                IGM.getPointerSize() * 2,
-                                IGM.getPointerAlignment(),
-                                std::move(spareBits),
-                                IsNotPOD);
+    if (T->isNoEscape()) {
+      // @noescape thick functions are trivial types.
+      return FuncTypeInfo::create(
+          CanSILFunctionType(T), IGM.NoEscapeFunctionPairTy,
+          IGM.getPointerSize() * 2, IGM.getPointerAlignment(),
+          std::move(spareBits), IsPOD);
+    }
+    return FuncTypeInfo::create(
+        CanSILFunctionType(T), IGM.FunctionPairTy, IGM.getPointerSize() * 2,
+        IGM.getPointerAlignment(), std::move(spareBits), IsNotPOD);
   }
   }
   llvm_unreachable("bad function type representation");
@@ -1171,8 +1175,10 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
     // If the parameters can live independent of the context, release it now
     // so we can tail call. The safety of this assumes that neither this release
     // nor any of the loads can throw.
-    if (consumesContext && !dependsOnContextLifetime && rawData)
+    if (consumesContext && !dependsOnContextLifetime && rawData) {
+      assert(!outType->isNoEscape() && "Trivial context must not be released");
       subIGF.emitNativeStrongRelease(rawData, subIGF.getDefaultAtomicity());
+    }
 
     // Now that we have bound generic parameters from the captured arguments
     // emit the polymorphic arguments.
@@ -1272,8 +1278,10 @@ static llvm::Function *emitPartialApplicationForwarder(IRGenModule &IGM,
   }
 
   // If the parameters depended on the context, consume the context now.
-  if (rawData && consumesContext && dependsOnContextLifetime)
+  if (rawData && consumesContext && dependsOnContextLifetime) {
+    assert(!outType->isNoEscape() && "Trivial context must not be released");
     subIGF.emitNativeStrongRelease(rawData, subIGF.getDefaultAtomicity());
+  }
 
   // Reabstract the result value as substituted.
   SILFunctionConventions origConv(origType, IGM.getSILModule());
@@ -1337,6 +1345,8 @@ void irgen::emitFunctionPartialApplication(
   SmallVector<SILType, 4> argValTypes;
   SmallVector<ParameterConvention, 4> argConventions;
 
+  bool isNoEscapeFunction = outType->isNoEscape();
+
   // Reserve space for polymorphic bindings.
   SubstitutionMap subMap;
   if (auto genericSig = origType->getGenericSignature())
@@ -1465,7 +1475,10 @@ void irgen::emitFunctionPartialApplication(
     fnPtr = IGF.Builder.CreateBitCast(fnPtr, IGF.IGM.Int8PtrTy);
     out.add(fnPtr);
     llvm::Value *ctx = args.claimNext();
-    ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.RefCountedPtrTy);
+    if (isNoEscapeFunction)
+      ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.OpaquePtrTy);
+    else
+      ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.RefCountedPtrTy);
     out.add(ctx);
     return;
   }
@@ -1506,7 +1519,10 @@ void irgen::emitFunctionPartialApplication(
     llvm::Value *ctx = args.claimNext();
     if (isIndirectFormalParameter(*singleRefcountedConvention))
       ctx = IGF.Builder.CreateLoad(ctx, IGF.IGM.getPointerAlignment());
-    ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.RefCountedPtrTy);
+    if (isNoEscapeFunction)
+      ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.OpaquePtrTy);
+    else
+      ctx = IGF.Builder.CreateBitCast(ctx, IGF.IGM.RefCountedPtrTy);
     out.add(ctx);
     return;
   }
@@ -1597,6 +1613,8 @@ void irgen::emitFunctionPartialApplication(
                                                               argConventions);
   forwarder = IGF.Builder.CreateBitCast(forwarder, IGF.IGM.Int8PtrTy);
   out.add(forwarder);
+  if (isNoEscapeFunction)
+    data = IGF.Builder.CreateBitCast(data, IGF.IGM.OpaquePtrTy);
   out.add(data);
 }
 

lib/IRGen/IRGenSIL.cpp

@@ -2090,7 +2090,7 @@ Callee LoweredValue::getCallee(IRGenFunction &IGF,
     case SILFunctionType::Representation::Closure:
     case SILFunctionType::Representation::Method:
       return getSwiftFunctionPointerCallee(IGF, functionValue, selfValue,
-                                           std::move(calleeInfo));
+                                           std::move(calleeInfo), false);
 
     case SILFunctionType::Representation::CFunctionPointer:
       assert(!selfValue && "C function pointer has self?");
@@ -2109,8 +2109,10 @@ Callee LoweredValue::getCallee(IRGenFunction &IGF,
     assert(vector.size() == 2 && "thick function pointer with size != 2");
     llvm::Value *functionValue = vector[0];
     llvm::Value *contextValue = vector[1];
+    bool castToRefcountedContext = calleeInfo.OrigFnType->isNoEscape();
     return getSwiftFunctionPointerCallee(IGF, functionValue, contextValue,
-                                         std::move(calleeInfo));
+                                         std::move(calleeInfo),
+                                         castToRefcountedContext);
   }
 
   case LoweredValue::Kind::EmptyExplosion:
@@ -4588,7 +4590,10 @@ void IRGenSILFunction::visitThinToThickFunctionInst(
   Explosion from = getLoweredExplosion(i->getOperand());
   Explosion to;
   to.add(from.claimNext());
-  to.add(IGM.RefCountedNull);
+  if (i->getType().castTo<SILFunctionType>()->isNoEscape())
+    to.add(llvm::ConstantPointerNull::get(IGM.OpaquePtrTy));
+  else
+    to.add(IGM.RefCountedNull);
   setLoweredExplosion(i, to);
 }
 

lib/IRGen/LoadableByAddress.cpp

@@ -2402,7 +2402,19 @@ void LoadableByAddress::recreateConvInstrs() {
           instr->getLoc(), instr->getOperand(), newType);
       break;
     }
-    default:
+    case SILInstructionKind::ConvertEscapeToNoEscapeInst: {
+      auto instr = cast<ConvertEscapeToNoEscapeInst>(convInstr);
+      newInstr = convBuilder.createConvertEscapeToNoEscape(
+          instr->getLoc(), instr->getOperand(), newType);
+      break;
+    }
+    case SILInstructionKind::MarkDependenceInst: {
+      auto instr = cast<MarkDependenceInst>(convInstr);
+      newInstr = convBuilder.createMarkDependence(
+          instr->getLoc(), instr->getValue(), instr->getBase());
+      break;
+    }
+     default:
       llvm_unreachable("Unexpected conversion instruction");
     }
     convInstr->replaceAllUsesWith(newInstr);
@@ -2484,6 +2496,8 @@ void LoadableByAddress::run() {
                 break;
               }
               case SILInstructionKind::ConvertFunctionInst:
+              case SILInstructionKind::ConvertEscapeToNoEscapeInst:
+              case SILInstructionKind::MarkDependenceInst:
               case SILInstructionKind::ThinFunctionToPointerInst:
               case SILInstructionKind::ThinToThickFunctionInst: {
                 conversionInstrs.insert(
@@ -2505,6 +2519,22 @@ void LoadableByAddress::run() {
             }
             funcRefs.insert(FRI);
           }
+        } else if (auto *Cvt = dyn_cast<MarkDependenceInst>(&I)) {
+          SILValue val = Cvt->getValue();
+          SILType currType = val->getType();
+          if (auto fType = currType.getAs<SILFunctionType>()) {
+            if (modifiableFunction(fType)) {
+              conversionInstrs.insert(Cvt);
+            }
+          }
+        } else if (auto *Cvt = dyn_cast<ConvertEscapeToNoEscapeInst>(&I)) {
+          SILValue val = Cvt->getConverted();
+          SILType currType = val->getType();
+          auto fType = currType.getAs<SILFunctionType>();
+          assert(fType && "Expected SILFunctionType");
+          if (modifiableFunction(fType)) {
+            conversionInstrs.insert(Cvt);
+          }
         } else if (auto *CFI = dyn_cast<ConvertFunctionInst>(&I)) {
           SILValue val = CFI->getConverted();
           SILType currType = val->getType();

lib/SIL/InstructionUtils.cpp

@@ -283,10 +283,15 @@ bool swift::onlyAffectsRefCount(SILInstruction *user) {
 
 SILValue swift::stripConvertFunctions(SILValue V) {
   while (true) {
-    auto CFI = dyn_cast<ConvertFunctionInst>(V);
-    if (!CFI)
-      return V;
-    V = CFI->getOperand();
+    if (auto CFI = dyn_cast<ConvertFunctionInst>(V)) {
+      V = CFI->getOperand();
+      continue;
+    }
+    else if (auto *Cvt = dyn_cast<ConvertEscapeToNoEscapeInst>(V)) {
+      V = Cvt->getOperand();
+      continue;
+    }
+    break;
   }
   return V;
 }
@@ -435,8 +440,10 @@ SILValue swift::isPartialApplyOfReabstractionThunk(PartialApplyInst *PAI) {
 
   // The argument should be a closure.
   auto Arg = PAI->getArgument(0);
-  if (!Arg->getType().is<SILFunctionType>()
-      || !Arg->getType().isReferenceCounted(PAI->getFunction()->getModule()))
+  if (!Arg->getType().is<SILFunctionType>() ||
+      (!Arg->getType().isReferenceCounted(PAI->getFunction()->getModule()) &&
+       Arg->getType().getAs<SILFunctionType>()->getRepresentation() !=
+           SILFunctionType::Representation::Thick))
     return SILValue();
 
   return Arg;

lib/SIL/SILFunctionType.cpp

@@ -2890,6 +2890,12 @@ SILFunctionType::isABICompatibleWith(CanSILFunctionType other) const {
       return {ABICompatibilityCheckResult::ABIIncompatibleParameterType, i};
   }
 
+  // This needs to be checked last because the result implies everying else has
+  // already been checked and this is the only difference.
+  if (isNoEscape() != other->isNoEscape() &&
+      (getRepresentation() == SILFunctionType::Representation::Thick))
+    return ABICompatibilityCheckResult::ABIEscapeToNoEscapeConversion;
+
   return ABICompatibilityCheckResult::None;
 }
 
@@ -2918,6 +2924,8 @@ StringRef SILFunctionType::ABICompatibilityCheckResult::getMessage() const {
     return "Differing parameter convention";
   case innerty::ABIIncompatibleParameterType:
     return "ABI incompatible parameter type.";
+  case innerty::ABIEscapeToNoEscapeConversion:
+    return "Escape to no escape conversion";
   }
   llvm_unreachable("Covered switch isn't completely covered?!");
 }

lib/SIL/SILInstructions.cpp

@@ -2014,8 +2014,9 @@ ConvertEscapeToNoEscapeInst *ConvertEscapeToNoEscapeInst::create(
     (void)opTI;
     CanSILFunctionType resTI = CFI->getType().castTo<SILFunctionType>();
     (void)resTI;
-    assert(opTI->isABICompatibleWith(resTI).isCompatible() &&
-           "Can not convert in between ABI incompatible function types");
+    assert(
+        opTI->isABICompatibleWith(resTI).isCompatibleUpToNoEscapeConversion() &&
+        "Can not convert in between ABI incompatible function types");
   }
   return CFI;
 }

lib/SIL/SILOwnershipVerifier.cpp

@@ -1008,6 +1008,9 @@ OwnershipUseCheckerResult OwnershipCompatibilityUseChecker::visitCallee(
     return {compatibleWithOwnership(ValueOwnershipKind::Owned),
             UseLifetimeConstraint::MustBeInvalidated};
   case ParameterConvention::Direct_Guaranteed:
+    if (SubstCalleeType->isNoEscape())
+      return {compatibleWithOwnership(ValueOwnershipKind::Trivial),
+        UseLifetimeConstraint::MustBeLive};
     return {compatibleWithOwnership(ValueOwnershipKind::Guaranteed),
             UseLifetimeConstraint::MustBeLive};
   }
@@ -1188,6 +1191,16 @@ OwnershipCompatibilityUseChecker::visitStoreInst(StoreInst *I) {
 OwnershipUseCheckerResult
 OwnershipCompatibilityUseChecker::visitMarkDependenceInst(
     MarkDependenceInst *MDI) {
+
+  // Forward ownership if the mark_dependence instruction marks a dependence
+  // on a @noescape function type for an escaping function type.
+  if (getValue() == MDI->getValue())
+    if (auto ResFnTy = MDI->getType().getAs<SILFunctionType>())
+      if (auto BaseFnTy = MDI->getBase()->getType().getAs<SILFunctionType>())
+        if (!ResFnTy->isNoEscape() && BaseFnTy->isNoEscape())
+          return {compatibleWithOwnership(ValueOwnershipKind::Owned),
+                  UseLifetimeConstraint::MustBeInvalidated};
+
   // We always treat mark dependence as a use that keeps a value alive. We will
   // be introducing a begin_dependence/end_dependence version of this later.
   return {true, UseLifetimeConstraint::MustBeLive};

lib/SIL/SILVerifier.cpp

@@ -2898,8 +2898,10 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
     require(resFTy->getRepresentation() == SILFunctionType::Representation::Thick,
             "result of thin_to_thick_function must be thick");
 
-    auto adjustedOperandExtInfo = opFTy->getExtInfo().withRepresentation(
-                                           SILFunctionType::Representation::Thick);
+    auto adjustedOperandExtInfo =
+        opFTy->getExtInfo()
+            .withRepresentation(SILFunctionType::Representation::Thick)
+            .withNoEscape(resFTy->isNoEscape());
     require(adjustedOperandExtInfo == resFTy->getExtInfo(),
             "operand and result of thin_to_think_function must agree in particulars");
   }