Alter the runtime interface for awaiting futures and task groups.

First, just call an async -> T function instead of forcing the caller
to piece together which case we're in and perform its own copy.  This
ensures that the task is actually kept alive properly.

Second, now that we no longer implicitly depend on the waiting tasks
being run synchronously, go ahead and schedule them to run on the
global executor.

This solves some problems which were blocking the work on TLS-ifying
the task/executor state.

Author: rjmccall

include/swift/ABI/Task.h

@@ -873,6 +873,10 @@ class AsyncTask : public HeapObject, public Job {
     /// Destroy the storage associated with the future.
     void destroy();
 
+    const Metadata *getResultType() const {
+      return resultType;
+    }
+
     /// Retrieve a pointer to the storage of result.
     OpaqueValue *getStoragePtr() {
       return reinterpret_cast<OpaqueValue *>(

include/swift/Runtime/Concurrency.h

@@ -117,37 +117,36 @@ SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
 JobPriority
 swift_task_escalate(AsyncTask *task, JobPriority newPriority);
 
-/// The result of waiting for a task future.
-struct TaskFutureWaitResult {
-  /// Whether the storage represents the error result vs. the successful
-  /// result.
-  bool hadErrorResult;
-
-  /// Storage for the result of the future.
-  ///
-  /// When the future completed normally, this is a pointer to the storage
-  /// of the result value, which lives inside the future task itself.
-  ///
-  /// When the future completed by throwing an error, this is the error
-  /// object itself.
-  OpaqueValue *storage;
-};
-
 using TaskFutureWaitSignature =
-  AsyncSignature<TaskFutureWaitResult(AsyncTask *), /*throws*/ false>;
+  AsyncSignature<void(AsyncTask *, OpaqueValue *), /*throws*/ false>;
 
-/// Wait for a future task to complete.
+/// Wait for a non-throwing future task to complete.
 ///
 /// This can be called from any thread. Its Swift signature is
 ///
 /// \code
-/// func swift_task_future_wait(on task: Builtin.NativeObject) async
-///     -> TaskFutureWaitResult
+/// func swift_task_future_wait(on task: _owned Builtin.NativeObject) async
+///     -> Success
 /// \endcode
 SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync)
 TaskFutureWaitSignature::FunctionType
 swift_task_future_wait;
 
+using TaskFutureWaitThrowingSignature =
+  AsyncSignature<void(AsyncTask *, OpaqueValue *), /*throws*/ true>;
+
+/// Wait for a potentially-throwing future task to complete.
+///
+/// This can be called from any thread. Its Swift signature is
+///
+/// \code
+/// func swift_task_future_wait_throwing(on task: _owned Builtin.NativeObject)
+///    async throws -> Success
+/// \endcode
+SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swiftasync)
+TaskFutureWaitThrowingSignature::FunctionType
+swift_task_future_wait_throwing;
+
 /// Wait for a readyQueue of a Channel to become non empty.
 ///
 /// This can be called from any thread. Its Swift signature is

lib/IRGen/Callee.h

@@ -125,28 +125,76 @@ namespace irgen {
   /// A function pointer value.
   class FunctionPointer {
   public:
-    struct KindTy {
-      enum class Value {
-        Function,
-        AsyncFunctionPointer,
-      };
-      static const Value Function = Value::Function;
-      static const Value AsyncFunctionPointer = Value::AsyncFunctionPointer;
-      Value value;
-      KindTy(Value value) : value(value) {}
-      KindTy(CanSILFunctionType fnType)
-          : value(fnType->isAsync() ? Value::AsyncFunctionPointer
-                                    : Value::Function) {}
-      friend bool operator==(const KindTy &lhs, const KindTy &rhs) {
+    enum class BasicKind {
+      Function,
+      AsyncFunctionPointer
+    };
+
+    enum class SpecialKind {
+      TaskFutureWait,
+      TaskFutureWaitThrowing,
+      TaskGroupWaitNext,
+    };
+
+    class Kind {
+      static constexpr unsigned SpecialOffset = 2;
+      unsigned value;
+    public:
+      static constexpr BasicKind Function =
+        BasicKind::Function;
+      static constexpr BasicKind AsyncFunctionPointer =
+        BasicKind::AsyncFunctionPointer;
+
+      Kind(BasicKind kind) : value(unsigned(kind)) {}
+      Kind(SpecialKind kind) : value(unsigned(kind) + SpecialOffset) {}
+      Kind(CanSILFunctionType fnType)
+        : Kind(fnType->isAsync() ? BasicKind::AsyncFunctionPointer
+                                 : BasicKind::Function) {}
+
+      BasicKind getBasicKind() const {
+        return value < SpecialOffset ? BasicKind(value) : BasicKind::Function;
+      }
+      bool isAsyncFunctionPointer() const {
+        return value == unsigned(BasicKind::AsyncFunctionPointer);
+      }
+
+      bool isSpecial() const {
+        return value >= SpecialOffset;
+      }
+      SpecialKind getSpecialKind() const {
+        assert(isSpecial());
+        return SpecialKind(value - SpecialOffset);
+      }
+
+      /// Should we suppress the generic signature from the given function?
+      ///
+      /// This is a micro-optimization we apply to certain special functions
+      /// that we know don't need generics.
+      bool suppressGenerics() const {
+        if (!isSpecial()) return false;
+
+        switch (getSpecialKind()) {
+        case SpecialKind::TaskFutureWait:
+        case SpecialKind::TaskFutureWaitThrowing:
+          // We suppress generics from these as a code-size optimization
+          // because the runtime can recover the success type from the
+          // future.
+          return true;
+        case SpecialKind::TaskGroupWaitNext:
+          return false;
+        }
+      }
+
+      friend bool operator==(Kind lhs, Kind rhs) {
         return lhs.value == rhs.value;
       }
-      friend bool operator!=(const KindTy &lhs, const KindTy &rhs) {
+      friend bool operator!=(Kind lhs, Kind rhs) {
         return !(lhs == rhs);
       }
     };
 
   private:
-    KindTy Kind;
+    Kind kind;
 
     /// The actual pointer, either to the function or to its descriptor.
     llvm::Value *Value;
@@ -155,34 +203,29 @@ namespace irgen {
 
     Signature Sig;
 
-    bool isFunctionPointerWithoutContext = false;
-
   public:
     /// Construct a FunctionPointer for an arbitrary pointer value.
     /// We may add more arguments to this; try to use the other
     /// constructors/factories if possible.
-    explicit FunctionPointer(KindTy kind, llvm::Value *value,
+    explicit FunctionPointer(Kind kind, llvm::Value *value,
                              PointerAuthInfo authInfo,
-                             const Signature &signature,
-                             bool isWithoutCtxt = false)
-        : Kind(kind), Value(value), AuthInfo(authInfo), Sig(signature),
-          isFunctionPointerWithoutContext(isWithoutCtxt) {
+                             const Signature &signature)
+        : kind(kind), Value(value), AuthInfo(authInfo), Sig(signature) {
       // The function pointer should have function type.
       assert(value->getType()->getPointerElementType()->isFunctionTy());
       // TODO: maybe assert similarity to signature.getType()?
     }
 
     // Temporary only!
-    explicit FunctionPointer(KindTy kind, llvm::Value *value,
-                             const Signature &signature,
-                             bool isWithoutCtxt = false)
-        : FunctionPointer(kind, value, PointerAuthInfo(), signature, isWithoutCtxt) {}
+    explicit FunctionPointer(Kind kind, llvm::Value *value,
+                             const Signature &signature)
+      : FunctionPointer(kind, value, PointerAuthInfo(), signature) {}
 
     static FunctionPointer forDirect(IRGenModule &IGM,
                                      llvm::Constant *value,
                                      CanSILFunctionType fnType);
 
-    static FunctionPointer forDirect(KindTy kind, llvm::Constant *value,
+    static FunctionPointer forDirect(Kind kind, llvm::Constant *value,
                                      const Signature &signature) {
       return FunctionPointer(kind, value, PointerAuthInfo(), signature);
     }
@@ -197,7 +240,8 @@ namespace irgen {
       return (isa<llvm::Constant>(Value) && AuthInfo.isConstant());
     }
 
-    KindTy getKind() const { return Kind; }
+    Kind getKind() const { return kind; }
+    BasicKind getBasicKind() const { return kind.getBasicKind(); }
 
     /// Given that this value is known to have been constructed from a direct
     /// function,  Return the name of that function.
@@ -250,7 +294,11 @@ namespace irgen {
     FunctionPointer getAsFunction(IRGenFunction &IGF) const;
 
     bool useStaticContextSize() const {
-      return isFunctionPointerWithoutContext;
+      return !kind.isAsyncFunctionPointer();
+    }
+
+    bool suppressGenerics() const {
+      return kind.suppressGenerics();
     }
   };
 
@@ -315,6 +363,10 @@ namespace irgen {
       return Fn.getSignature();
     }
 
+    bool suppressGenerics() const {
+      return Fn.suppressGenerics();
+    }
+
     /// If this callee has a value for the Swift context slot, return
     /// it; otherwise return non-null.
     llvm::Value *getSwiftContext() const;