[Async CC] Make error indirect.

include/swift/ABI/Task.h

@@ -600,8 +600,10 @@ class AsyncTask : public HeapObject, public Job {
     /// The type of the result that will be produced by the future.
     const Metadata *resultType;
 
-    // Trailing storage for the result itself. The storage will be uninitialized,
-    // contain an instance of \c resultType, or contain an an \c Error.
+    SwiftError *error = nullptr;
+
+    // Trailing storage for the result itself. The storage will be
+    // uninitialized, contain an instance of \c resultType.
 
     friend class AsyncTask;
 
@@ -620,25 +622,20 @@ class AsyncTask : public HeapObject, public Job {
     }
 
     /// Retrieve the error.
-    SwiftError *&getError() {
-      return *reinterpret_cast<SwiftError **>(
-           reinterpret_cast<char *>(this) + storageOffset(resultType));
-    }
+    SwiftError *&getError() { return *&error; }
 
     /// Compute the offset of the storage from the base of the future
     /// fragment.
     static size_t storageOffset(const Metadata *resultType)  {
       size_t offset = sizeof(FutureFragment);
-      size_t alignment =
-          std::max(resultType->vw_alignment(), alignof(SwiftError *));
+      size_t alignment = resultType->vw_alignment();
       return (offset + alignment - 1) & ~(alignment - 1);
     }
 
     /// Determine the size of the future fragment given a particular future
     /// result type.
     static size_t fragmentSize(const Metadata *resultType) {
-      return storageOffset(resultType) +
-          std::max(resultType->vw_size(), sizeof(SwiftError *));
+      return storageOffset(resultType) + resultType->vw_size();
     }
   };
 
@@ -791,7 +788,7 @@ class YieldingAsyncContext : public AsyncContext {
 /// futures.
 class FutureAsyncContext : public AsyncContext {
 public:
-  SwiftError *errorResult = nullptr;
+  SwiftError **errorResult = nullptr;
   OpaqueValue *indirectResult;
 
   using AsyncContext::AsyncContext;

lib/IRGen/CallEmission.h

@@ -95,7 +95,7 @@ class CallEmission {
   /// Set the arguments to the function from an explosion.
   virtual void setArgs(Explosion &arg, bool isOutlined,
                        WitnessMetadata *witnessMetadata);
-  virtual Address getCalleeErrorSlot(SILType errorType) = 0;
+  virtual Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) = 0;
 
   void addAttribute(unsigned Index, llvm::Attribute::AttrKind Attr);
 

lib/IRGen/GenCall.cpp

@@ -150,10 +150,11 @@ irgen::getAsyncContextLayout(IRGenModule &IGM, CanSILFunctionType originalType,
     addTaskContinuationFunction();
   }
 
-  //   SwiftError *errorResult;
+  //   SwiftError **errorResult;
   auto errorCanType = IGM.Context.getExceptionType();
   auto errorType = SILType::getPrimitiveObjectType(errorCanType);
-  auto &errorTypeInfo = IGM.getTypeInfoForLowered(errorCanType);
+  auto &errorTypeInfo =
+      IGM.getTypeInfoForLowered(CanInOutType::get(errorCanType));
   typeInfos.push_back(&errorTypeInfo);
   valTypes.push_back(errorType);
 
@@ -2320,7 +2321,7 @@ class SyncCallEmission final : public CallEmission {
 
     out = nativeSchema.mapFromNative(IGF.IGM, IGF, nativeExplosion, resultType);
   }
-  Address getCalleeErrorSlot(SILType errorType) override {
+  Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) override {
     return IGF.getCalleeErrorResultSlot(errorType);
   };
 };
@@ -2378,10 +2379,10 @@ class AsyncCallEmission final : public CallEmission {
     context = layout.emitCastTo(IGF, contextBuffer.getAddress());
     if (layout.canHaveError()) {
       auto fieldLayout = layout.getErrorLayout();
-      auto addr = fieldLayout.project(IGF, context, /*offsets*/ llvm::None);
-      auto &ti = fieldLayout.getType();
-      auto nullError = llvm::Constant::getNullValue(ti.getStorageType());
-      IGF.Builder.CreateStore(nullError, addr);
+      auto ptrToAddr =
+          fieldLayout.project(IGF, context, /*offsets*/ llvm::None);
+      auto errorSlot = IGF.getAsyncCalleeErrorResultSlot(layout.getErrorType());
+      IGF.Builder.CreateStore(errorSlot.getAddress(), ptrToAddr);
     }
   }
   void end() override {
@@ -2504,11 +2505,18 @@ class AsyncCallEmission final : public CallEmission {
       loadValue(fieldLayout, out);
     }
   }
-  Address getCalleeErrorSlot(SILType errorType) override {
-    auto layout = getAsyncContextLayout();
-    auto errorLayout = layout.getErrorLayout();
-    auto address = errorLayout.project(IGF, context, /*offsets*/ llvm::None);
-    return address;
+  Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) override {
+    if (isCalleeAsync) {
+      auto layout = getAsyncContextLayout();
+      auto errorLayout = layout.getErrorLayout();
+      auto pointerToAddress =
+          errorLayout.project(IGF, context, /*offsets*/ llvm::None);
+      auto load = IGF.Builder.CreateLoad(pointerToAddress);
+      auto address = Address(load, IGF.IGM.getPointerAlignment());
+      return address;
+    } else {
+      return IGF.getCalleeErrorResultSlot(errorType);
+    }
   };
 
   llvm::CallInst *createCall(const FunctionPointer &fn,
@@ -3931,42 +3939,60 @@ Explosion IRGenFunction::collectParameters() {
   return params;
 }
 
-/// Fetch the error result slot.
-Address IRGenFunction::getCalleeErrorResultSlot(SILType errorType) {
-  if (!CalleeErrorResultSlot) {
-    auto &errorTI = cast<FixedTypeInfo>(getTypeInfo(errorType));
+Address IRGenFunction::createErrorResultSlot(SILType errorType, bool isAsync) {
+  auto &errorTI = cast<FixedTypeInfo>(getTypeInfo(errorType));
 
-    IRBuilder builder(IGM.getLLVMContext(), IGM.DebugInfo != nullptr);
-    builder.SetInsertPoint(AllocaIP->getParent(), AllocaIP->getIterator());
+  IRBuilder builder(IGM.getLLVMContext(), IGM.DebugInfo != nullptr);
+  builder.SetInsertPoint(AllocaIP->getParent(), AllocaIP->getIterator());
 
-    // Create the alloca.  We don't use allocateStack because we're
-    // not allocating this in stack order.
-    auto addr = createAlloca(errorTI.getStorageType(),
-                             errorTI.getFixedAlignment(),
-                             "swifterror");
+  // Create the alloca.  We don't use allocateStack because we're
+  // not allocating this in stack order.
+  auto addr = createAlloca(errorTI.getStorageType(),
+                           errorTI.getFixedAlignment(), "swifterror");
 
-    // Only add the swifterror attribute on ABIs that pass it in a register.
-    // We create a shadow stack location of the swifterror parameter for the
-    // debugger on platforms that pass swifterror by reference and so we can't
-    // mark the parameter with a swifterror attribute for these.
-    if (IGM.IsSwiftErrorInRegister)
-      cast<llvm::AllocaInst>(addr.getAddress())->setSwiftError(true);
+  // Only add the swifterror attribute on ABIs that pass it in a register.
+  // We create a shadow stack location of the swifterror parameter for the
+  // debugger on platforms that pass swifterror by reference and so we can't
+  // mark the parameter with a swifterror attribute for these.
+  // The slot for async callees cannot be annotated swifterror because those
+  // errors are never passed in registers but rather are always passed
+  // indirectly in the async context.
+  if (IGM.IsSwiftErrorInRegister && !isAsync)
+    cast<llvm::AllocaInst>(addr.getAddress())->setSwiftError(true);
+
+  // Initialize at the alloca point.
+  auto nullError = llvm::ConstantPointerNull::get(
+      cast<llvm::PointerType>(errorTI.getStorageType()));
+  builder.CreateStore(nullError, addr);
 
-    // Initialize at the alloca point.
-    auto nullError = llvm::ConstantPointerNull::get(
-                            cast<llvm::PointerType>(errorTI.getStorageType()));
-    builder.CreateStore(nullError, addr);
+  return addr;
+}
 
-    CalleeErrorResultSlot = addr.getAddress();
+/// Fetch the error result slot.
+Address IRGenFunction::getCalleeErrorResultSlot(SILType errorType) {
+  if (!CalleeErrorResultSlot) {
+    CalleeErrorResultSlot =
+        createErrorResultSlot(errorType, /*isAsync=*/false).getAddress();
   }
   return Address(CalleeErrorResultSlot, IGM.getPointerAlignment());
 }
 
+/// Fetch the error result slot.
+Address IRGenFunction::getAsyncCalleeErrorResultSlot(SILType errorType) {
+  assert(isAsync() &&
+         "throwing async functions must be called from async functions");
+  if (!AsyncCalleeErrorResultSlot) {
+    AsyncCalleeErrorResultSlot =
+        createErrorResultSlot(errorType, /*isAsync=*/true).getAddress();
+  }
+  return Address(AsyncCalleeErrorResultSlot, IGM.getPointerAlignment());
+}
+
 /// Fetch the error result slot received from the caller.
 Address IRGenFunction::getCallerErrorResultSlot() {
   assert(CallerErrorResultSlot && "no error result slot!");
   assert(isa<llvm::Argument>(CallerErrorResultSlot) && !isAsync() ||
-         isa<llvm::GetElementPtrInst>(CallerErrorResultSlot) && isAsync() &&
+         isa<llvm::LoadInst>(CallerErrorResultSlot) && isAsync() &&
              "error result slot is local!");
   return Address(CallerErrorResultSlot, IGM.getPointerAlignment());
 }

lib/IRGen/GenCall.h

@@ -70,7 +70,7 @@ namespace irgen {
   //   SwiftPartialFunction * __ptrauth(...) returnToCaller;
   //   SwiftActor * __ptrauth(...) callerActor;
   //   SwiftPartialFunction * __ptrauth(...) yieldToCaller?;
-  //   SwiftError *errorResult;
+  //   SwiftError **errorResult;
   //   IndirectResultTypes *indirectResults...;
   //   union {
   //     struct {

lib/IRGen/GenThunk.cpp

@@ -153,8 +153,11 @@ void IRGenThunk::prepareArguments() {
     }
 
     if (origTy->hasErrorResult()) {
-      Address addr = asyncLayout->getErrorLayout().project(
-          IGF, context, llvm::None);
+      auto errorLayout = asyncLayout->getErrorLayout();
+      Address pointerToAddress =
+          errorLayout.project(IGF, context, /*offsets*/ llvm::None);
+      auto load = IGF.Builder.CreateLoad(pointerToAddress);
+      auto addr = Address(load, IGF.IGM.getPointerAlignment());
       IGF.setCallerErrorResultSlot(addr.getAddress());
     }
 
@@ -334,7 +337,8 @@ void IRGenThunk::emit() {
 
   if (isAsync && origTy->hasErrorResult()) {
     SILType errorType = conv.getSILErrorType(expansionContext);
-    Address calleeErrorSlot = emission->getCalleeErrorSlot(errorType);
+    Address calleeErrorSlot = emission->getCalleeErrorSlot(
+        errorType, /*isCalleeAsync=*/origTy->isAsync());
     errorValue = IGF.Builder.CreateLoad(calleeErrorSlot);
   }
 

lib/IRGen/IRGenFunction.h

@@ -89,7 +89,10 @@ class IRGenFunction {
 
   friend class Scope;
 
-//--- Function prologue and epilogue -------------------------------------------
+  Address createErrorResultSlot(SILType errorType, bool isAsync);
+
+  //--- Function prologue and epilogue
+  //-------------------------------------------
 public:
   Explosion collectParameters();
   void emitScalarReturn(SILType returnResultType, SILType funcResultType,
@@ -106,6 +109,7 @@ class IRGenFunction {
   /// For async functions, this is different from the caller result slot because
   /// that is a gep into the %swift.context.
   Address getCalleeErrorResultSlot(SILType errorType);
+  Address getAsyncCalleeErrorResultSlot(SILType errorType);
 
   /// Return the error result slot provided by the caller.
   Address getCallerErrorResultSlot();
@@ -165,6 +169,7 @@ class IRGenFunction {
   Address ReturnSlot;
   llvm::BasicBlock *ReturnBB;
   llvm::Value *CalleeErrorResultSlot = nullptr;
+  llvm::Value *AsyncCalleeErrorResultSlot = nullptr;
   llvm::Value *CallerErrorResultSlot = nullptr;
   llvm::Value *CoroutineHandle = nullptr;
   llvm::Value *AsyncCoroutineCurrentResume = nullptr;

lib/IRGen/IRGenSIL.cpp

@@ -1295,7 +1295,10 @@ class AsyncNativeCCEntryPointArgumentEmission final
 
   llvm::Value *getCallerErrorResultArgument() override {
     auto errorLayout = layout.getErrorLayout();
-    Address addr = errorLayout.project(IGF, dataAddr, /*offsets*/ llvm::None);
+    Address pointerToAddress =
+        errorLayout.project(IGF, dataAddr, /*offsets*/ llvm::None);
+    auto load = IGF.Builder.CreateLoad(pointerToAddress);
+    auto addr = Address(load, IGF.IGM.getPointerAlignment());
     return addr.getAddress();
   }
   llvm::Value *getContext() override {
@@ -2837,7 +2840,8 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
     SILFunctionConventions substConv(substCalleeType, getSILModule());
     SILType errorType =
         substConv.getSILErrorType(IGM.getMaximalTypeExpansionContext());
-    Address calleeErrorSlot = emission->getCalleeErrorSlot(errorType);
+    Address calleeErrorSlot = emission->getCalleeErrorSlot(
+        errorType, /*isCalleeAsync=*/site.getOrigCalleeType()->isAsync());
     auto errorValue = Builder.CreateLoad(calleeErrorSlot);
     emission->end();
 

stdlib/public/Concurrency/AsyncCall.h

@@ -63,12 +63,12 @@ struct AsyncFrameLayout;
 
 template <class... ArgTys, bool HasErrorResult>
 struct AsyncFrameLayout<AsyncSignature<void(ArgTys...), HasErrorResult>> {
-  using BasicLayout = BasicLayout<0, SwiftError*, ArgTys...>;
+  using BasicLayout = BasicLayout<0, SwiftError **, ArgTys...>;
   static constexpr size_t firstArgIndex = 1;
 };
 template <class ResultTy, class... ArgTys, bool HasErrorResult>
 struct AsyncFrameLayout<AsyncSignature<ResultTy(ArgTys...), HasErrorResult>> {
-  using BasicLayout = BasicLayout<0, SwiftError*, ResultTy, ArgTys...>;
+  using BasicLayout = BasicLayout<0, SwiftError **, ResultTy, ArgTys...>;
   static constexpr size_t firstArgIndex = 2;
 };
 

stdlib/public/Concurrency/Task.cpp

@@ -94,7 +94,7 @@ void AsyncTask::completeFuture(AsyncContext *context, ExecutorRef executor) {
   // If an error was thrown, save it in the future fragment.
   auto futureContext = static_cast<FutureAsyncContext *>(context);
   bool hadErrorResult = false;
-  if (auto errorObject = futureContext->errorResult) {
+  if (auto errorObject = *futureContext->errorResult) {
     fragment->getError() = errorObject;
     hadErrorResult = true;
   }
@@ -283,7 +283,7 @@ AsyncTaskAndContext swift::swift_task_create_future_f(
     // Set up the context for the future so there is no error, and a successful
     // result will be written into the future fragment's storage.
     auto futureContext = static_cast<FutureAsyncContext *>(initialContext);
-    futureContext->errorResult = nullptr;
+    futureContext->errorResult = &futureFragment->getError();
     futureContext->indirectResult = futureFragment->getStoragePtr();
   }
 

stdlib/public/Concurrency/TaskGroup.cpp

@@ -74,7 +74,7 @@ void AsyncTask::groupOffer(AsyncTask *completedTask, AsyncContext *context,
   // If an error was thrown, save it in the future fragment.
   auto futureContext = static_cast<FutureAsyncContext *>(context);
   bool hadErrorResult = false;
-  if (auto errorObject = futureContext->errorResult) {
+  if (auto errorObject = *futureContext->errorResult) {
     // instead we need to enqueue this result:
     hadErrorResult = true;
   }

stdlib/public/Concurrency/TaskPrivate.h

@@ -60,7 +60,7 @@ namespace {
 class TaskFutureWaitAsyncContext : public AsyncContext {
 public:
   // Error result is always present.
-  SwiftError *errorResult = nullptr;
+  SwiftError **errorResult = nullptr;
 
   // No indirect results.
 

test/IRGen/async/hop_to_executor.sil

@@ -14,8 +14,8 @@ final actor MyActor {
 // CHECK-LABEL: define{{.*}} void @test_simple(%swift.task* %0, %swift.executor* %1, %swift.context* swiftasync %2)
 // CHECK: [[TASK_LOC:%[0-9]+]] = alloca %swift.task*
 // CHECK: [[CTX:%[0-9]+]] = bitcast %swift.context* %2
-// CHECK-32: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, %swift.error*, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 5
-// CHECK-64: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, [4 x i8], %swift.error*, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 6
+// CHECK-32: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, %swift.error**, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 5
+// CHECK-64: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, [4 x i8], %swift.error**, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 6
 // CHECK: [[ACTOR:%[0-9]+]] = load %T4test7MyActorC*, %T4test7MyActorC** [[ACTOR_ADDR]]
 // CHECK: [[RESUME:%[0-9]+]] = call i8* @llvm.coro.async.resume()
 // CHECK: [[TASK:%[0-9]+]] = load %swift.task*, %swift.task** [[TASK_LOC]]