Skip to content

Commit de63a23

Browse files
nate-chandlernate-chandler
authored
Merge pull request #35603 from nate-chandler/concurrency/irgen/rdar71378532
[Async CC] Make error indirect.
2 parents 5e851ac + e2a8abc commit de63a23

File tree

13 files changed

+251
-62
lines changed

13 files changed

+251
-62
lines changed

‎include/swift/ABI/Task.h

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -845,8 +845,10 @@ class AsyncTask : public HeapObject, public Job {
845845
/// The type of the result that will be produced by the future.
846846
const Metadata *resultType;
847847

848-
// Trailing storage for the result itself. The storage will be uninitialized,
849-
// contain an instance of \c resultType, or contain an an \c Error.
848+
SwiftError *error = nullptr;
849+
850+
// Trailing storage for the result itself. The storage will be
851+
// uninitialized, contain an instance of \c resultType.
850852

851853
friend class AsyncTask;
852854

@@ -865,25 +867,20 @@ class AsyncTask : public HeapObject, public Job {
865867
}
866868

867869
/// Retrieve the error.
868-
SwiftError *&getError() {
869-
return *reinterpret_cast<SwiftError **>(
870-
reinterpret_cast<char *>(this) + storageOffset(resultType));
871-
}
870+
SwiftError *&getError() { return *&error; }
872871

873872
/// Compute the offset of the storage from the base of the future
874873
/// fragment.
875874
static size_t storageOffset(const Metadata *resultType) {
876875
size_t offset = sizeof(FutureFragment);
877-
size_t alignment =
878-
std::max(resultType->vw_alignment(), alignof(SwiftError *));
876+
size_t alignment = resultType->vw_alignment();
879877
return (offset + alignment - 1) & ~(alignment - 1);
880878
}
881879

882880
/// Determine the size of the future fragment given a particular future
883881
/// result type.
884882
static size_t fragmentSize(const Metadata *resultType) {
885-
return storageOffset(resultType) +
886-
std::max(resultType->vw_size(), sizeof(SwiftError *));
883+
return storageOffset(resultType) + resultType->vw_size();
887884
}
888885
};
889886

@@ -1038,7 +1035,7 @@ class YieldingAsyncContext : public AsyncContext {
10381035
/// futures.
10391036
class FutureAsyncContext : public AsyncContext {
10401037
public:
1041-
SwiftError *errorResult = nullptr;
1038+
SwiftError **errorResult = nullptr;
10421039
OpaqueValue *indirectResult;
10431040

10441041
using AsyncContext::AsyncContext;

‎lib/IRGen/CallEmission.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ class CallEmission {
9595
/// Set the arguments to the function from an explosion.
9696
virtual void setArgs(Explosion &arg, bool isOutlined,
9797
WitnessMetadata *witnessMetadata);
98-
virtual Address getCalleeErrorSlot(SILType errorType) = 0;
98+
virtual Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) = 0;
9999

100100
void addAttribute(unsigned Index, llvm::Attribute::AttrKind Attr);
101101

‎lib/IRGen/GenCall.cpp

Lines changed: 61 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,11 @@ irgen::getAsyncContextLayout(IRGenModule &IGM, CanSILFunctionType originalType,
150150
addTaskContinuationFunction();
151151
}
152152

153-
// SwiftError *errorResult;
153+
// SwiftError **errorResult;
154154
auto errorCanType = IGM.Context.getExceptionType();
155155
auto errorType = SILType::getPrimitiveObjectType(errorCanType);
156-
auto &errorTypeInfo = IGM.getTypeInfoForLowered(errorCanType);
156+
auto &errorTypeInfo =
157+
IGM.getTypeInfoForLowered(CanInOutType::get(errorCanType));
157158
typeInfos.push_back(&errorTypeInfo);
158159
valTypes.push_back(errorType);
159160

@@ -2330,7 +2331,7 @@ class SyncCallEmission final : public CallEmission {
23302331

23312332
out = nativeSchema.mapFromNative(IGF.IGM, IGF, nativeExplosion, resultType);
23322333
}
2333-
Address getCalleeErrorSlot(SILType errorType) override {
2334+
Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) override {
23342335
return IGF.getCalleeErrorResultSlot(errorType);
23352336
};
23362337
};
@@ -2388,10 +2389,10 @@ class AsyncCallEmission final : public CallEmission {
23882389
context = layout.emitCastTo(IGF, contextBuffer.getAddress());
23892390
if (layout.canHaveError()) {
23902391
auto fieldLayout = layout.getErrorLayout();
2391-
auto addr = fieldLayout.project(IGF, context, /*offsets*/ llvm::None);
2392-
auto &ti = fieldLayout.getType();
2393-
auto nullError = llvm::Constant::getNullValue(ti.getStorageType());
2394-
IGF.Builder.CreateStore(nullError, addr);
2392+
auto ptrToAddr =
2393+
fieldLayout.project(IGF, context, /*offsets*/ llvm::None);
2394+
auto errorSlot = IGF.getAsyncCalleeErrorResultSlot(layout.getErrorType());
2395+
IGF.Builder.CreateStore(errorSlot.getAddress(), ptrToAddr);
23952396
}
23962397
}
23972398
void end() override {
@@ -2514,11 +2515,18 @@ class AsyncCallEmission final : public CallEmission {
25142515
loadValue(fieldLayout, out);
25152516
}
25162517
}
2517-
Address getCalleeErrorSlot(SILType errorType) override {
2518-
auto layout = getAsyncContextLayout();
2519-
auto errorLayout = layout.getErrorLayout();
2520-
auto address = errorLayout.project(IGF, context, /*offsets*/ llvm::None);
2521-
return address;
2518+
Address getCalleeErrorSlot(SILType errorType, bool isCalleeAsync) override {
2519+
if (isCalleeAsync) {
2520+
auto layout = getAsyncContextLayout();
2521+
auto errorLayout = layout.getErrorLayout();
2522+
auto pointerToAddress =
2523+
errorLayout.project(IGF, context, /*offsets*/ llvm::None);
2524+
auto load = IGF.Builder.CreateLoad(pointerToAddress);
2525+
auto address = Address(load, IGF.IGM.getPointerAlignment());
2526+
return address;
2527+
} else {
2528+
return IGF.getCalleeErrorResultSlot(errorType);
2529+
}
25222530
};
25232531

25242532
llvm::CallInst *createCall(const FunctionPointer &fn,
@@ -3945,42 +3953,60 @@ Explosion IRGenFunction::collectParameters() {
39453953
return params;
39463954
}
39473955

3948-
/// Fetch the error result slot.
3949-
Address IRGenFunction::getCalleeErrorResultSlot(SILType errorType) {
3950-
if (!CalleeErrorResultSlot) {
3951-
auto &errorTI = cast<FixedTypeInfo>(getTypeInfo(errorType));
3956+
Address IRGenFunction::createErrorResultSlot(SILType errorType, bool isAsync) {
3957+
auto &errorTI = cast<FixedTypeInfo>(getTypeInfo(errorType));
39523958

3953-
IRBuilder builder(IGM.getLLVMContext(), IGM.DebugInfo != nullptr);
3954-
builder.SetInsertPoint(AllocaIP->getParent(), AllocaIP->getIterator());
3959+
IRBuilder builder(IGM.getLLVMContext(), IGM.DebugInfo != nullptr);
3960+
builder.SetInsertPoint(AllocaIP->getParent(), AllocaIP->getIterator());
39553961

3956-
// Create the alloca. We don't use allocateStack because we're
3957-
// not allocating this in stack order.
3958-
auto addr = createAlloca(errorTI.getStorageType(),
3959-
errorTI.getFixedAlignment(),
3960-
"swifterror");
3962+
// Create the alloca. We don't use allocateStack because we're
3963+
// not allocating this in stack order.
3964+
auto addr = createAlloca(errorTI.getStorageType(),
3965+
errorTI.getFixedAlignment(), "swifterror");
39613966

3962-
// Only add the swifterror attribute on ABIs that pass it in a register.
3963-
// We create a shadow stack location of the swifterror parameter for the
3964-
// debugger on platforms that pass swifterror by reference and so we can't
3965-
// mark the parameter with a swifterror attribute for these.
3966-
if (IGM.IsSwiftErrorInRegister)
3967-
cast<llvm::AllocaInst>(addr.getAddress())->setSwiftError(true);
3967+
// Only add the swifterror attribute on ABIs that pass it in a register.
3968+
// We create a shadow stack location of the swifterror parameter for the
3969+
// debugger on platforms that pass swifterror by reference and so we can't
3970+
// mark the parameter with a swifterror attribute for these.
3971+
// The slot for async callees cannot be annotated swifterror because those
3972+
// errors are never passed in registers but rather are always passed
3973+
// indirectly in the async context.
3974+
if (IGM.IsSwiftErrorInRegister && !isAsync)
3975+
cast<llvm::AllocaInst>(addr.getAddress())->setSwiftError(true);
3976+
3977+
// Initialize at the alloca point.
3978+
auto nullError = llvm::ConstantPointerNull::get(
3979+
cast<llvm::PointerType>(errorTI.getStorageType()));
3980+
builder.CreateStore(nullError, addr);
39683981

3969-
// Initialize at the alloca point.
3970-
auto nullError = llvm::ConstantPointerNull::get(
3971-
cast<llvm::PointerType>(errorTI.getStorageType()));
3972-
builder.CreateStore(nullError, addr);
3982+
return addr;
3983+
}
39733984

3974-
CalleeErrorResultSlot = addr.getAddress();
3985+
/// Fetch the error result slot.
3986+
Address IRGenFunction::getCalleeErrorResultSlot(SILType errorType) {
3987+
if (!CalleeErrorResultSlot) {
3988+
CalleeErrorResultSlot =
3989+
createErrorResultSlot(errorType, /*isAsync=*/false).getAddress();
39753990
}
39763991
return Address(CalleeErrorResultSlot, IGM.getPointerAlignment());
39773992
}
39783993

3994+
/// Fetch the error result slot.
3995+
Address IRGenFunction::getAsyncCalleeErrorResultSlot(SILType errorType) {
3996+
assert(isAsync() &&
3997+
"throwing async functions must be called from async functions");
3998+
if (!AsyncCalleeErrorResultSlot) {
3999+
AsyncCalleeErrorResultSlot =
4000+
createErrorResultSlot(errorType, /*isAsync=*/true).getAddress();
4001+
}
4002+
return Address(AsyncCalleeErrorResultSlot, IGM.getPointerAlignment());
4003+
}
4004+
39794005
/// Fetch the error result slot received from the caller.
39804006
Address IRGenFunction::getCallerErrorResultSlot() {
39814007
assert(CallerErrorResultSlot && "no error result slot!");
39824008
assert(isa<llvm::Argument>(CallerErrorResultSlot) && !isAsync() ||
3983-
isa<llvm::GetElementPtrInst>(CallerErrorResultSlot) && isAsync() &&
4009+
isa<llvm::LoadInst>(CallerErrorResultSlot) && isAsync() &&
39844010
"error result slot is local!");
39854011
return Address(CallerErrorResultSlot, IGM.getPointerAlignment());
39864012
}

‎lib/IRGen/GenCall.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ namespace irgen {
7070
// SwiftPartialFunction * __ptrauth(...) returnToCaller;
7171
// SwiftActor * __ptrauth(...) callerActor;
7272
// SwiftPartialFunction * __ptrauth(...) yieldToCaller?;
73-
// SwiftError *errorResult;
73+
// SwiftError **errorResult;
7474
// IndirectResultTypes *indirectResults...;
7575
// union {
7676
// struct {

‎lib/IRGen/GenThunk.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,11 @@ void IRGenThunk::prepareArguments() {
154154
}
155155

156156
if (origTy->hasErrorResult()) {
157-
Address addr = asyncLayout->getErrorLayout().project(
158-
IGF, context, llvm::None);
157+
auto errorLayout = asyncLayout->getErrorLayout();
158+
Address pointerToAddress =
159+
errorLayout.project(IGF, context, /*offsets*/ llvm::None);
160+
auto load = IGF.Builder.CreateLoad(pointerToAddress);
161+
auto addr = Address(load, IGF.IGM.getPointerAlignment());
159162
IGF.setCallerErrorResultSlot(addr.getAddress());
160163
}
161164

@@ -338,7 +341,8 @@ void IRGenThunk::emit() {
338341

339342
if (isAsync && origTy->hasErrorResult()) {
340343
SILType errorType = conv.getSILErrorType(expansionContext);
341-
Address calleeErrorSlot = emission->getCalleeErrorSlot(errorType);
344+
Address calleeErrorSlot = emission->getCalleeErrorSlot(
345+
errorType, /*isCalleeAsync=*/origTy->isAsync());
342346
errorValue = IGF.Builder.CreateLoad(calleeErrorSlot);
343347
}
344348

‎lib/IRGen/IRGenFunction.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,10 @@ class IRGenFunction {
8989

9090
friend class Scope;
9191

92-
//--- Function prologue and epilogue -------------------------------------------
92+
Address createErrorResultSlot(SILType errorType, bool isAsync);
93+
94+
//--- Function prologue and epilogue
95+
//-------------------------------------------
9396
public:
9497
Explosion collectParameters();
9598
void emitScalarReturn(SILType returnResultType, SILType funcResultType,
@@ -106,6 +109,7 @@ class IRGenFunction {
106109
/// For async functions, this is different from the caller result slot because
107110
/// that is a gep into the %swift.context.
108111
Address getCalleeErrorResultSlot(SILType errorType);
112+
Address getAsyncCalleeErrorResultSlot(SILType errorType);
109113

110114
/// Return the error result slot provided by the caller.
111115
Address getCallerErrorResultSlot();
@@ -165,6 +169,7 @@ class IRGenFunction {
165169
Address ReturnSlot;
166170
llvm::BasicBlock *ReturnBB;
167171
llvm::Value *CalleeErrorResultSlot = nullptr;
172+
llvm::Value *AsyncCalleeErrorResultSlot = nullptr;
168173
llvm::Value *CallerErrorResultSlot = nullptr;
169174
llvm::Value *CoroutineHandle = nullptr;
170175
llvm::Value *AsyncCoroutineCurrentResume = nullptr;

‎lib/IRGen/IRGenSIL.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1301,7 +1301,10 @@ class AsyncNativeCCEntryPointArgumentEmission final
13011301

13021302
llvm::Value *getCallerErrorResultArgument() override {
13031303
auto errorLayout = layout.getErrorLayout();
1304-
Address addr = errorLayout.project(IGF, dataAddr, /*offsets*/ llvm::None);
1304+
Address pointerToAddress =
1305+
errorLayout.project(IGF, dataAddr, /*offsets*/ llvm::None);
1306+
auto load = IGF.Builder.CreateLoad(pointerToAddress);
1307+
auto addr = Address(load, IGF.IGM.getPointerAlignment());
13051308
return addr.getAddress();
13061309
}
13071310
llvm::Value *getContext() override {
@@ -2839,7 +2842,8 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
28392842
SILFunctionConventions substConv(substCalleeType, getSILModule());
28402843
SILType errorType =
28412844
substConv.getSILErrorType(IGM.getMaximalTypeExpansionContext());
2842-
Address calleeErrorSlot = emission->getCalleeErrorSlot(errorType);
2845+
Address calleeErrorSlot = emission->getCalleeErrorSlot(
2846+
errorType, /*isCalleeAsync=*/site.getOrigCalleeType()->isAsync());
28432847
auto errorValue = Builder.CreateLoad(calleeErrorSlot);
28442848
emission->end();
28452849

‎stdlib/public/Concurrency/AsyncCall.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,12 @@ struct AsyncFrameLayout;
6363

6464
template <class... ArgTys, bool HasErrorResult>
6565
struct AsyncFrameLayout<AsyncSignature<void(ArgTys...), HasErrorResult>> {
66-
using BasicLayout = BasicLayout<0, SwiftError*, ArgTys...>;
66+
using BasicLayout = BasicLayout<0, SwiftError **, ArgTys...>;
6767
static constexpr size_t firstArgIndex = 1;
6868
};
6969
template <class ResultTy, class... ArgTys, bool HasErrorResult>
7070
struct AsyncFrameLayout<AsyncSignature<ResultTy(ArgTys...), HasErrorResult>> {
71-
using BasicLayout = BasicLayout<0, SwiftError*, ResultTy, ArgTys...>;
71+
using BasicLayout = BasicLayout<0, SwiftError **, ResultTy, ArgTys...>;
7272
static constexpr size_t firstArgIndex = 2;
7373
};
7474

‎stdlib/public/Concurrency/Task.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ void AsyncTask::completeFuture(AsyncContext *context, ExecutorRef executor) {
9696
// If an error was thrown, save it in the future fragment.
9797
auto futureContext = static_cast<FutureAsyncContext *>(context);
9898
bool hadErrorResult = false;
99-
if (auto errorObject = futureContext->errorResult) {
99+
if (auto errorObject = *futureContext->errorResult) {
100100
fragment->getError() = errorObject;
101101
hadErrorResult = true;
102102
}
@@ -294,7 +294,7 @@ AsyncTaskAndContext swift::swift_task_create_future_f(
294294
// Set up the context for the future so there is no error, and a successful
295295
// result will be written into the future fragment's storage.
296296
auto futureContext = static_cast<FutureAsyncContext *>(initialContext);
297-
futureContext->errorResult = nullptr;
297+
futureContext->errorResult = &futureFragment->getError();
298298
futureContext->indirectResult = futureFragment->getStoragePtr();
299299
}
300300

‎stdlib/public/Concurrency/TaskGroup.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ void AsyncTask::groupOffer(AsyncTask *completedTask, AsyncContext *context,
7474
// If an error was thrown, save it in the future fragment.
7575
auto futureContext = static_cast<FutureAsyncContext *>(context);
7676
bool hadErrorResult = false;
77-
if (auto errorObject = futureContext->errorResult) {
77+
if (auto errorObject = *futureContext->errorResult) {
7878
// instead we need to enqueue this result:
7979
hadErrorResult = true;
8080
}

‎stdlib/public/Concurrency/TaskPrivate.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ namespace {
6060
class TaskFutureWaitAsyncContext : public AsyncContext {
6161
public:
6262
// Error result is always present.
63-
SwiftError *errorResult = nullptr;
63+
SwiftError **errorResult = nullptr;
6464

6565
// No indirect results.
6666

‎test/IRGen/async/hop_to_executor.sil

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ final actor MyActor {
1414
// CHECK-LABEL: define{{.*}} void @test_simple(%swift.task* %0, %swift.executor* %1, %swift.context* swiftasync %2)
1515
// CHECK: [[TASK_LOC:%[0-9]+]] = alloca %swift.task*
1616
// CHECK: [[CTX:%[0-9]+]] = bitcast %swift.context* %2
17-
// CHECK-32: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, %swift.error*, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 5
18-
// CHECK-64: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, [4 x i8], %swift.error*, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 6
17+
// CHECK-32: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, %swift.error**, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 5
18+
// CHECK-64: [[ACTOR_ADDR:%[0-9]+]] = getelementptr inbounds <{ %swift.context*, void (%swift.task*, %swift.executor*, %swift.context*)*, %swift.executor*, i32, [4 x i8], %swift.error**, %T4test7MyActorC* }>, {{.*}} [[CTX]], i32 0, i32 6
1919
// CHECK: [[ACTOR:%[0-9]+]] = load %T4test7MyActorC*, %T4test7MyActorC** [[ACTOR_ADDR]]
2020
// CHECK: [[RESUME:%[0-9]+]] = call i8* @llvm.coro.async.resume()
2121
// CHECK: [[TASK:%[0-9]+]] = load %swift.task*, %swift.task** [[TASK_LOC]]

0 commit comments

Comments
 (0)