SILGen support for throwing indirect errors

SwiftCompilerSources/Sources/SIL/Instruction.swift

@@ -1094,6 +1094,10 @@ final public class ThrowInst : TermInst, UnaryInstruction {
   public override var isFunctionExiting: Bool { true }
 }
 
+final public class ThrowAddrInst : TermInst {
+  public override var isFunctionExiting: Bool { true }
+}
+
 final public class YieldInst : TermInst {
 }
 

SwiftCompilerSources/Sources/SIL/Registration.swift

@@ -172,6 +172,7 @@ public func registerSILClasses() {
   register(UnreachableInst.self)
   register(ReturnInst.self)
   register(ThrowInst.self)
+  register(ThrowAddrInst.self)
   register(YieldInst.self)
   register(UnwindInst.self)
   register(TryApplyInst.self)

docs/SIL.rst

@@ -613,8 +613,8 @@ the caller.  A non-autoreleased ``apply`` of a function that is defined
 with an autoreleased result has the effect of performing an
 autorelease in the callee.
 
-- SIL function types may provide an optional error result, written by
-  placing ``@error`` on a result.  An error result is always
+- SIL function types may provide an optional direct error result, written by
+  placing ``@error`` on a result.  A direct error result is always
   implicitly ``@owned``.  Only functions with a native calling
   convention may have an error result.
 
@@ -8006,6 +8006,28 @@ the basic block argument will be the operand of the ``throw``.
 
 A function must not contain more than one ``throw`` instruction.
 
+throw_addr
+``````````
+::
+
+  sil-terminator ::= 'throw_addr'
+
+  throw_addr
+  // indirect error result must be initialized at this point
+
+Exits the current function and returns control to the calling
+function. The current function must have an indirect error result,
+and so the function must have been invoked with a ``try_apply``
+instruction. Control will resume in the error destination of
+that instruction.
+
+The function is responsible for initializing its error result
+before the ``throw_addr``. 
+
+``throw_addr`` does not retain or release any values.
+
+A function must not contain more than one ``throw_addr`` instruction.
+
 yield
 `````
 ::

include/swift/SIL/AddressWalker.h

@@ -150,6 +150,7 @@ TransitiveAddressWalker<Impl>::walk(SILValue projectedAddress) && {
 
       case TermKind::UnreachableInst:
       case TermKind::UnwindInst:
+      case TermKind::ThrowAddrInst:
         llvm_unreachable("Should never be used");
       case TermKind::SwitchEnumInst:
       case TermKind::SwitchValueInst:

include/swift/SIL/ApplySite.h

@@ -741,6 +741,14 @@ class FullApplySite : public ApplySite {
       && (getCalleeArgIndex(op) < getNumIndirectSILResults());
   }
 
+  /// Returns true if \p op is an operand that passes an indirect
+  /// result argument to the apply site.
+  bool isIndirectErrorResultOperand(const Operand &op) const {
+    return isArgumentOperand(op)
+      && (getCalleeArgIndex(op) >= getNumIndirectSILResults())
+      && (getCalleeArgIndex(op) < getNumIndirectSILErrorResults());
+  }
+
   static FullApplySite getFromOpaqueValue(void *p) { return FullApplySite(p); }
 
   static bool classof(const SILInstruction *inst) {

include/swift/SIL/SILBuilder.h

@@ -2484,6 +2484,11 @@ class SILBuilder {
         new (getModule()) ThrowInst(getSILDebugLocation(Loc), errorValue));
   }
 
+  ThrowAddrInst *createThrowAddr(SILLocation Loc) {
+    return insertTerminator(
+        new (getModule()) ThrowAddrInst(getSILDebugLocation(Loc)));
+  }
+
   UnwindInst *createUnwind(SILLocation loc) {
     return insertTerminator(
         new (getModule()) UnwindInst(getSILDebugLocation(loc)));

include/swift/SIL/SILCloner.h

@@ -3134,6 +3134,14 @@ SILCloner<ImplClass>::visitThrowInst(ThrowInst *Inst) {
                                      getOpValue(Inst->getOperand())));
 }
 
+template<typename ImplClass>
+void
+SILCloner<ImplClass>::visitThrowAddrInst(ThrowAddrInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(
+      Inst, getBuilder().createThrowAddr(getOpLocation(Inst->getLoc())));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitUnwindInst(UnwindInst *Inst) {

include/swift/SIL/SILInstruction.h

@@ -9168,6 +9168,7 @@ class TermInst : public NonValueInstruction {
     case TermKind::UnreachableInst:
     case TermKind::ReturnInst:
     case TermKind::ThrowInst:
+    case TermKind::ThrowAddrInst:
     case TermKind::YieldInst:
     case TermKind::CondBranchInst:
     case TermKind::BranchInst:
@@ -9295,8 +9296,7 @@ class ReturnInst
   }
 };
 
-/// ThrowInst - Throw a typed error (which, in our system, is
-/// essentially just a funny kind of return).
+/// ThrowInst - Throw a typed error, returning it via the direct error result.
 class ThrowInst
   : public UnaryInstructionBase<SILInstructionKind::ThrowInst, TermInst>
 {
@@ -9317,6 +9317,30 @@ class ThrowInst
   }
 };
 
+/// ThrowAddrInst - Throw a typed error, previously stored in the indirect
+/// error result.
+class ThrowAddrInst
+  : public InstructionBase<SILInstructionKind::ThrowAddrInst, TermInst>
+{
+  friend SILBuilder;
+
+  /// Constructs a ThrowAddrInst representing a throw out of the current
+  /// function.
+  ///
+  /// \param DebugLoc The location of the throw.
+  ThrowAddrInst(SILDebugLocation DebugLoc)
+      : InstructionBase(DebugLoc) {}
+
+public:
+  SuccessorListTy getSuccessors() {
+    // No successors.
+    return SuccessorListTy();
+  }
+
+  ArrayRef<Operand> getAllOperands() const { return {}; }
+  MutableArrayRef<Operand> getAllOperands() { return {}; }
+};
+
 /// UnwindInst - Continue unwinding out of this function.  Currently this is
 /// only used in coroutines as the eventual terminator of the unwind edge
 /// out of a 'yield'.

include/swift/SIL/SILNodes.def

@@ -722,6 +722,8 @@ ABSTRACT_INST(TermInst, SILInstruction)
              TermInst, None, DoesNotRelease)
   TERMINATOR(ThrowInst, throw,
              TermInst, None, DoesNotRelease)
+  TERMINATOR(ThrowAddrInst, throw_addr,
+             TermInst, None, DoesNotRelease)
   TERMINATOR(YieldInst, yield,
              TermInst, MayHaveSideEffects, MayRelease)
   TERMINATOR(UnwindInst, unwind,

include/swift/SILOptimizer/Utils/SCCVisitor.h

@@ -132,6 +132,7 @@ class SCCVisitor {
     case TermKind::ReturnInst:
     case TermKind::SwitchValueInst:
     case TermKind::ThrowInst:
+    case TermKind::ThrowAddrInst:
     case TermKind::UnwindInst:
       llvm_unreachable("Did not expect terminator that does not have args!");
 

lib/IRGen/IRGenSIL.cpp

@@ -1469,6 +1469,7 @@ class IRGenSILFunction :
   void visitCondBranchInst(CondBranchInst *i);
   void visitReturnInst(ReturnInst *i);
   void visitThrowInst(ThrowInst *i);
+  void visitThrowAddrInst(ThrowAddrInst *i);
   void visitUnwindInst(UnwindInst *i);
   void visitYieldInst(YieldInst *i);
   void visitSwitchValueInst(SwitchValueInst *i);
@@ -4251,23 +4252,34 @@ void IRGenSILFunction::visitReturnInst(swift::ReturnInst *i) {
 void IRGenSILFunction::visitThrowInst(swift::ThrowInst *i) {
   SILFunctionConventions conv(CurSILFn->getLoweredFunctionType(),
                               getSILModule());
+  assert(!conv.hasIndirectSILErrorResults());
+
   if (!isAsync()) {
     if (conv.isTypedError()) {
       llvm::Constant *flag = llvm::ConstantInt::get(IGM.IntPtrTy, 1);
       flag = llvm::ConstantExpr::getIntToPtr(flag, IGM.Int8PtrTy);
-      if (!conv.hasIndirectSILErrorResults()) {
-        Explosion errorResult = getLoweredExplosion(i->getOperand());
-        auto &ti = cast<LoadableTypeInfo>(IGM.getTypeInfo(conv.getSILErrorType(
-              IGM.getMaximalTypeExpansionContext())));
-        ti.initialize(*this, errorResult, getCallerTypedErrorResultSlot(), false);
-      }
+      Explosion errorResult = getLoweredExplosion(i->getOperand());
+      auto &ti = cast<LoadableTypeInfo>(IGM.getTypeInfo(conv.getSILErrorType(
+            IGM.getMaximalTypeExpansionContext())));
+      ti.initialize(*this, errorResult, getCallerTypedErrorResultSlot(), false);
+
       Builder.CreateStore(flag, getCallerErrorResultSlot());
     } else {
       Explosion errorResult = getLoweredExplosion(i->getOperand());
       Builder.CreateStore(errorResult.claimNext(), getCallerErrorResultSlot());
     }
-    // Async functions just return to the continuation.
-  } else if (isAsync()) {
+
+    // Create a normal return, but leaving the return value undefined.
+    auto fnTy = CurFn->getFunctionType();
+    auto retTy = fnTy->getReturnType();
+    if (retTy->isVoidTy()) {
+      Builder.CreateRetVoid();
+    } else {
+      Builder.CreateRet(llvm::UndefValue::get(retTy));
+    }
+
+  // Async functions just return to the continuation.
+  } else {
     // Store the exception to the error slot.
     auto exn = getLoweredExplosion(i->getOperand());
 
@@ -4276,13 +4288,9 @@ void IRGenSILFunction::visitThrowInst(swift::ThrowInst *i) {
         conv.getSILResultType(IGM.getMaximalTypeExpansionContext()));
 
     if (conv.isTypedError()) {
-      if (conv.hasIndirectSILErrorResults()) {
-        (void)exn.claimAll();
-      } else {
-        auto &ti = cast<LoadableTypeInfo>(IGM.getTypeInfo(conv.getSILErrorType(
-              IGM.getMaximalTypeExpansionContext())));
-        ti.initialize(*this, exn, getCallerTypedErrorResultSlot(), false);
-      }
+      auto &ti = cast<LoadableTypeInfo>(IGM.getTypeInfo(conv.getSILErrorType(
+            IGM.getMaximalTypeExpansionContext())));
+      ti.initialize(*this, exn, getCallerTypedErrorResultSlot(), false);
       llvm::Constant *flag = llvm::ConstantInt::get(IGM.IntPtrTy, 1);
       flag = llvm::ConstantExpr::getIntToPtr(flag, IGM.Int8PtrTy);
       assert(exn.empty() && "Unclaimed typed error results");
@@ -4293,16 +4301,44 @@ void IRGenSILFunction::visitThrowInst(swift::ThrowInst *i) {
     Explosion empty;
     emitAsyncReturn(*this, layout, funcResultType,
                     i->getFunction()->getLoweredFunctionType(), empty, exn);
-    return;
   }
+}
 
-  // Create a normal return, but leaving the return value undefined.
-  auto fnTy = CurFn->getFunctionType();
-  auto retTy = fnTy->getReturnType();
-  if (retTy->isVoidTy()) {
-    Builder.CreateRetVoid();
+void IRGenSILFunction::visitThrowAddrInst(swift::ThrowAddrInst *i) {
+  SILFunctionConventions conv(CurSILFn->getLoweredFunctionType(),
+                              getSILModule());
+  assert(conv.isTypedError());
+  assert(conv.hasIndirectSILErrorResults());
+
+  if (!isAsync()) {
+    llvm::Constant *flag = llvm::ConstantInt::get(IGM.IntPtrTy, 1);
+    flag = llvm::ConstantExpr::getIntToPtr(flag, IGM.Int8PtrTy);
+    Builder.CreateStore(flag, getCallerErrorResultSlot());
+
+    // Create a normal return, but leaving the return value undefined.
+    auto fnTy = CurFn->getFunctionType();
+    auto retTy = fnTy->getReturnType();
+    if (retTy->isVoidTy()) {
+      Builder.CreateRetVoid();
+    } else {
+      Builder.CreateRet(llvm::UndefValue::get(retTy));
+    }
+
+  // Async functions just return to the continuation.
   } else {
-    Builder.CreateRet(llvm::UndefValue::get(retTy));
+    auto layout = getAsyncContextLayout(*this);
+    auto funcResultType = CurSILFn->mapTypeIntoContext(
+        conv.getSILResultType(IGM.getMaximalTypeExpansionContext()));
+
+    llvm::Constant *flag = llvm::ConstantInt::get(IGM.IntPtrTy, 1);
+    flag = llvm::ConstantExpr::getIntToPtr(flag, IGM.Int8PtrTy);
+
+    Explosion exn;
+    exn.add(flag);
+
+    Explosion empty;
+    emitAsyncReturn(*this, layout, funcResultType,
+                    i->getFunction()->getLoweredFunctionType(), empty, exn);
   }
 }
 

lib/SIL/IR/OperandOwnership.cpp

@@ -120,6 +120,7 @@ SHOULD_NEVER_VISIT_INST(StringLiteral)
 SHOULD_NEVER_VISIT_INST(StrongRetain)
 SHOULD_NEVER_VISIT_INST(Unreachable)
 SHOULD_NEVER_VISIT_INST(Unwind)
+SHOULD_NEVER_VISIT_INST(ThrowAddr)
 SHOULD_NEVER_VISIT_INST(ReleaseValue)
 SHOULD_NEVER_VISIT_INST(ReleaseValueAddr)
 SHOULD_NEVER_VISIT_INST(StrongRelease)

lib/SIL/IR/SILArgument.cpp

@@ -312,6 +312,7 @@ getSingleTerminatorOperandForPred(const SILBasicBlock *parentBlock,
   case TermKind::UnreachableInst:
   case TermKind::ReturnInst:
   case TermKind::ThrowInst:
+  case TermKind::ThrowAddrInst:
   case TermKind::UnwindInst:
     llvm_unreachable("Have terminator that implies no successors?!");
   case TermKind::TryApplyInst:

lib/SIL/IR/SILInstructions.cpp

@@ -1699,6 +1699,7 @@ bool TermInst::isFunctionExiting() const {
     return false;
   case TermKind::ReturnInst:
   case TermKind::ThrowInst:
+  case TermKind::ThrowAddrInst:
   case TermKind::UnwindInst:
     return true;
   }
@@ -1719,6 +1720,7 @@ bool TermInst::isProgramTerminating() const {
   case TermKind::CheckedCastAddrBranchInst:
   case TermKind::ReturnInst:
   case TermKind::ThrowInst:
+  case TermKind::ThrowAddrInst:
   case TermKind::UnwindInst:
   case TermKind::TryApplyInst:
   case TermKind::YieldInst:
@@ -1745,6 +1747,7 @@ const Operand *TermInst::forwardedOperand() const {
   case TermKind::UnreachableInst:
   case TermKind::ReturnInst:
   case TermKind::ThrowInst:
+  case TermKind::ThrowAddrInst:
   case TermKind::YieldInst:
   case TermKind::TryApplyInst:
   case TermKind::CondBranchInst:

lib/SIL/IR/SILPrinter.cpp

@@ -2625,6 +2625,10 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
     *this << getIDAndType(TI->getOperand());
   }
 
+  void visitThrowAddrInst(ThrowAddrInst *TAI) {
+    // no operands
+  }
+
   void visitUnwindInst(UnwindInst *UI) {
     // no operands
   }

lib/SIL/Parser/ParseSIL.cpp

@@ -5457,6 +5457,12 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
       ResultVal = B.createThrow(InstLoc, Val);
       break;
     }
+    case SILInstructionKind::ThrowAddrInst: {
+      if (parseSILDebugLocation(InstLoc, B))
+        return true;
+      ResultVal = B.createThrowAddr(InstLoc);
+      break;
+    }
     case SILInstructionKind::UnwindInst: {
       if (parseSILDebugLocation(InstLoc, B))
         return true;

lib/SIL/Utils/BasicBlockUtils.cpp

@@ -232,6 +232,7 @@ void swift::getEdgeArgs(TermInst *T, unsigned edgeIdx, SILBasicBlock *newEdgeBB,
 
   case SILInstructionKind::ReturnInst:
   case SILInstructionKind::ThrowInst:
+  case SILInstructionKind::ThrowAddrInst:
   case SILInstructionKind::UnwindInst:
   case SILInstructionKind::UnreachableInst:
     llvm_unreachable("terminator never has successors");

lib/SIL/Utils/InstructionUtils.cpp

@@ -524,6 +524,7 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType)
   case SILInstructionKind::UnreachableInst:
   case SILInstructionKind::ReturnInst:
   case SILInstructionKind::ThrowInst:
+  case SILInstructionKind::ThrowAddrInst:
   case SILInstructionKind::YieldInst:
   case SILInstructionKind::UnwindInst:
   case SILInstructionKind::BranchInst: