Merge branch 'main' of github.com:apple/swift into tensorflow-stage

* 'main' of github.com:apple/swift: (89 commits)
  Fix SILVerifier assert for witness_method with dynamic self type (swiftlang#34738)
  Allow a borrowed address to be passed as in_guaranteed arg in partial_apply [on_stack] only (swiftlang#34736)
  Fix test for windows/linux
  IRGen: Fix debug emission for dynamically sized stack vars
  [Testing] Add missing REQUIRES: concurrency
  [ownership] Make SILUndef always have ValueOwnershipKind::None.
  [value-lifetime] Cleanup constructors.
  [ownership] Centralize the concept of isReborrow on BorrowingOperand.
  [sil][value-lifetime] Add ValueLifetimeAnalysis::FrontierImpl = SmallVectorImpl<SILInstruction *>
  [Test] Disable partial_apply_forwarder.sil for arm64e.
  [Async CC] Unroll workaround where @main was async.
  Remove test that depends on Swift.Int metadata, which isn't around.
  [Concurrency] More cleanups for futures
  [Concurrency] Remove an unnecessary 'const' from a non-static data member.
  [Concurrency] Minor cleanups to futures.
  [Concurrency] Have future functions write their results directly.
  [Concurrency] Clarify/fix error object handling in futures.
  [Concurrency] Inline AsyncTask::FutureFragment::fragmentSize.
  [Concurrency] Turn "simple task" into just "task".
  [Concurrency] Use SchedulerPrivate for the "next waiting task" link.
  ...

Author: ainu-bot

CHANGELOG.md

@@ -6,6 +6,7 @@ CHANGELOG
 
 | Version                | Released   | Toolchain   |
 | :--------------------- | :--------- | :---------- |
+| [Swift 5.4](#swift-54) |            |             |
 | [Swift 5.3](#swift-53) | 2020-09-16 | Xcode 12.0  |
 | [Swift 5.2](#swift-52) | 2020-03-24 | Xcode 11.4  |
 | [Swift 5.1](#swift-51) | 2019-09-20 | Xcode 11.0  |
@@ -24,8 +25,8 @@ CHANGELOG
 
 </details>
 
-Swift Next
-----------
+Swift 5.4
+---------
 
 * [SR-10069][]:
 

docs/ABI/Mangling.rst

@@ -220,7 +220,8 @@ types where the metadata itself has unknown layout.)
   global ::= entity                      // some identifiable thing
   global ::= from-type to-type generic-signature? 'TR'  // reabstraction thunk
   global ::= from-type to-type generic-signature? 'TR'  // reabstraction thunk
-  global ::= impl-function-type 'Tz'     // objc-to-swift-async completion handler block implementation
+  global ::= impl-function-type type 'Tz'     // objc-to-swift-async completion handler block implementation
+  global ::= impl-function-type type 'TZ'     // objc-to-swift-async completion handler block implementation (predefined by runtime)
   global ::= from-type to-type self-type generic-signature? 'Ty'  // reabstraction thunk with dynamic 'Self' capture
   global ::= from-type to-type generic-signature? 'Tr'  // obsolete mangling for reabstraction thunk
   global ::= entity generic-signature? type type* 'TK' // key path getter

include/swift/ABI/MetadataKind.def

@@ -85,8 +85,8 @@ METADATAKIND(HeapGenericLocalVariable,
 METADATAKIND(ErrorObject,
              1 | MetadataKindIsNonType | MetadataKindIsRuntimePrivate)
 
-/// A heap-allocated simple task.
-METADATAKIND(SimpleTask,
+/// A heap-allocated task.
+METADATAKIND(Task,
              2 | MetadataKindIsNonType | MetadataKindIsRuntimePrivate)
 
 // getEnumeratedMetadataKind assumes that all the enumerated values here

include/swift/ABI/Task.h

@@ -19,6 +19,7 @@
 
 #include "swift/Basic/RelativePointer.h"
 #include "swift/ABI/HeapObject.h"
+#include "swift/ABI/Metadata.h"
 #include "swift/ABI/MetadataValues.h"
 #include "swift/Runtime/Config.h"
 #include "swift/Basic/STLExtras.h"
@@ -29,6 +30,8 @@ class AsyncTask;
 class AsyncContext;
 class Executor;
 class Job;
+struct OpaqueValue;
+struct SwiftError;
 class TaskStatusRecord;
 
 /// An ExecutorRef isn't necessarily just a pointer to an executor
@@ -86,6 +89,13 @@ class AsyncFunctionPointer {
 
 /// A schedulable job.
 class alignas(2 * alignof(void*)) Job {
+protected:
+  // Indices into SchedulerPrivate, for use by the runtime.
+  enum {
+    /// The next waiting task link, an AsyncTask that is waiting on a future.
+    NextWaitingTaskIndex = 0,
+  };
+
 public:
   // Reserved for the use of the scheduler.
   void *SchedulerPrivate[2];
@@ -230,19 +240,142 @@ class AsyncTask : public HeapObject, public Job {
     }
   };
 
-  bool isFuture() const { return Flags.task_isFuture(); }
-
   bool hasChildFragment() const { return Flags.task_isChildTask(); }
   ChildFragment *childFragment() {
     assert(hasChildFragment());
     return reinterpret_cast<ChildFragment*>(this + 1);
   }
 
-  // TODO: Future fragment
+  class FutureFragment {
+  public:
+    /// Describes the status of the future.
+    ///
+    /// Futures always begin in the "Executing" state, and will always
+    /// make a single state change to either Success or Error.
+    enum class Status : uintptr_t {
+      /// The future is executing or ready to execute. The storage
+      /// is not accessible.
+      Executing = 0,
+
+      /// The future has completed with result (of type \c resultType).
+      Success,
+
+      /// The future has completed by throwing an error (an \c Error
+      /// existential).
+      Error,
+    };
+
+    /// An item within the wait queue, which includes the status and the
+    /// head of the list of tasks.
+    struct WaitQueueItem {
+      /// Mask used for the low status bits in a wait queue item.
+      static const uintptr_t statusMask = 0x03;
+
+      uintptr_t storage;
+
+      Status getStatus() const {
+        return static_cast<Status>(storage & statusMask);
+      }
+
+      AsyncTask *getTask() const {
+        return reinterpret_cast<AsyncTask *>(storage & ~statusMask);
+      }
+
+      static WaitQueueItem get(Status status, AsyncTask *task) {
+        return WaitQueueItem{
+          reinterpret_cast<uintptr_t>(task) | static_cast<uintptr_t>(status)};
+      }
+    };
+
+  private:
+    /// Queue containing all of the tasks that are waiting in `get()`.
+    ///
+    /// The low bits contain the status, the rest of the pointer is the
+    /// AsyncTask.
+    std::atomic<WaitQueueItem> waitQueue;
+
+    /// The type of the result that will be produced by the future.
+    const Metadata *resultType;
+
+    // Trailing storage for the result itself. The storage will be uninitialized,
+    // contain an instance of \c resultType, or contaon an an \c Error.
+
+    friend class AsyncTask;
+
+  public:
+    explicit FutureFragment(const Metadata *resultType)
+      : waitQueue(WaitQueueItem::get(Status::Executing, nullptr)),
+        resultType(resultType) { }
+
+    /// Destroy the storage associated with the future.
+    void destroy();
+
+    /// Retrieve a pointer to the storage of result.
+    OpaqueValue *getStoragePtr() {
+      return reinterpret_cast<OpaqueValue *>(
+          reinterpret_cast<char *>(this) + storageOffset(resultType));
+    }
+
+    /// Retrieve the error.
+    SwiftError *&getError() {
+      return *reinterpret_cast<SwiftError **>(
+           reinterpret_cast<char *>(this) + storageOffset(resultType));
+    }
+
+    /// Compute the offset of the storage from the base of the future
+    /// fragment.
+    static size_t storageOffset(const Metadata *resultType)  {
+      size_t offset = sizeof(FutureFragment);
+      size_t alignment =
+          std::max(resultType->vw_alignment(), alignof(SwiftError *));
+      return (offset + alignment - 1) & ~(alignment - 1);
+    }
+
+    /// Determine the size of the future fragment given a particular future
+    /// result type.
+    static size_t fragmentSize(const Metadata *resultType) {
+      return storageOffset(resultType) +
+          std::max(resultType->vw_size(), sizeof(SwiftError *));
+    }
+  };
+
+  bool isFuture() const { return Flags.task_isFuture(); }
+
+  FutureFragment *futureFragment() {
+    assert(isFuture());
+    if (hasChildFragment()) {
+      return reinterpret_cast<FutureFragment *>(
+          reinterpret_cast<ChildFragment*>(this + 1) + 1);
+    }
+
+    return reinterpret_cast<FutureFragment *>(this + 1);
+  }
+
+  /// Wait for this future to complete.
+  ///
+  /// \returns the status of the future. If this result is
+  /// \c Executing, then \c waitingTask has been added to the
+  /// wait queue and will be scheduled when the future completes. Otherwise,
+  /// the future has completed and can be queried.
+  FutureFragment::Status waitFuture(AsyncTask *waitingTask);
+
+  /// Complete this future.
+  ///
+  /// Upon completion, any waiting tasks will be scheduled on the given
+  /// executor.
+  void completeFuture(AsyncContext *context, ExecutorRef executor);
 
   static bool classof(const Job *job) {
     return job->isAsyncTask();
   }
+
+private:
+  /// Access the next waiting task, which establishes a singly linked list of
+  /// tasks that are waiting on a future.
+  AsyncTask *&getNextWaitingTask() {
+    return reinterpret_cast<AsyncTask *&>(
+        SchedulerPrivate[NextWaitingTaskIndex]);
+  }
 };
 
 // The compiler will eventually assume these.
@@ -327,6 +460,20 @@ class YieldingAsyncContext : public AsyncContext {
   }
 };
 
+/// An asynchronous context within a task that describes a general "Future".
+/// task.
+///
+/// This type matches the ABI of a function `<T> () async throws -> T`, which
+/// is the type used by `Task.runDetached` and `Task.group.add` to create
+/// futures.
+class FutureAsyncContext : public AsyncContext {
+public:
+  SwiftError *errorResult = nullptr;
+  OpaqueValue *indirectResult;
+
+  using AsyncContext::AsyncContext;
+};
+
 } // end namespace swift
 
 #endif

include/swift/AST/ASTContext.h

@@ -672,6 +672,14 @@ class ASTContext final {
   /// if applicable.
   const Decl *getSwiftDeclForExportedClangDecl(const clang::Decl *decl);
 
+  /// General conversion method from Swift types -> Clang types.
+  ///
+  /// HACK: This method is only intended to be called from a specific place in
+  /// IRGen. For converting function types, strongly prefer using one of the
+  /// other methods instead, instead of manually iterating over parameters
+  /// and results.
+  const clang::Type *getClangTypeForIRGen(Type ty);
+
   /// Determine whether the given Swift type is representable in a
   /// given foreign language.
   ForeignRepresentationInfo
@@ -933,6 +941,13 @@ class ASTContext final {
   /// \returns The requested module, or NULL if the module cannot be found.
   ModuleDecl *getModule(ImportPath::Module ModulePath);
 
+  /// Attempts to load the matching overlay module for the given clang
+  /// module into this ASTContext.
+  ///
+  /// \returns The Swift overlay module corresponding to the given Clang module,
+  /// or NULL if the overlay module cannot be found.
+  ModuleDecl *getOverlayModule(const FileUnit *ClangModule);
+
   ModuleDecl *getModuleByName(StringRef ModuleName);
 
   ModuleDecl *getModuleByIdentifier(Identifier ModuleID);

include/swift/AST/ASTMangler.h

@@ -155,6 +155,15 @@ class ASTMangler : public Mangler {
                                              Type SelfType,
                                              ModuleDecl *Module);
 
+  /// Mangle a completion handler block implementation function, used for importing ObjC
+  /// APIs as async.
+  ///
+  /// - If `predefined` is true, this mangles the symbol name of the completion handler
+  /// predefined in the Swift runtime for the given type signature.
+  std::string mangleObjCAsyncCompletionHandlerImpl(CanSILFunctionType BlockType,
+                                                   CanType ResultType,
+                                                   bool predefined);
+  
   /// Mangle the derivative function (JVP/VJP) for the given:
   /// - Mangled original function name.
   /// - Derivative function kind.

include/swift/AST/Decl.h

@@ -391,7 +391,7 @@ class alignas(1 << DeclAlignInBits) Decl {
   SWIFT_INLINE_BITFIELD(SubscriptDecl, VarDecl, 2,
     StaticSpelling : 2
   );
-  SWIFT_INLINE_BITFIELD(AbstractFunctionDecl, ValueDecl, 3+8+1+1+1+1+1+1,
+  SWIFT_INLINE_BITFIELD(AbstractFunctionDecl, ValueDecl, 3+8+1+1+1+1+1+1+1,
     /// \see AbstractFunctionDecl::BodyKind
     BodyKind : 3,
 
@@ -415,7 +415,11 @@ class alignas(1 << DeclAlignInBits) Decl {
     Synthesized : 1,
 
     /// Whether this member's body consists of a single expression.
-    HasSingleExpressionBody : 1
+    HasSingleExpressionBody : 1,
+
+    /// Whether peeking into this function detected nested type declarations.
+    /// This is set when skipping over the decl at parsing.
+    HasNestedTypeDeclarations : 1
   );
 
   SWIFT_INLINE_BITFIELD(FuncDecl, AbstractFunctionDecl, 1+1+2+1+1+2+1,
@@ -5567,6 +5571,7 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
     Bits.AbstractFunctionDecl.Throws = Throws;
     Bits.AbstractFunctionDecl.Synthesized = false;
     Bits.AbstractFunctionDecl.HasSingleExpressionBody = false;
+    Bits.AbstractFunctionDecl.HasNestedTypeDeclarations = false;
   }
 
   void setBodyKind(BodyKind K) {
@@ -5713,6 +5718,16 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
     setBody(S, BodyKind::Parsed);
   }
 
+  /// Was there a nested type declaration detected when parsing this
+  /// function was skipped?
+  bool hasNestedTypeDeclarations() const {
+    return Bits.AbstractFunctionDecl.HasNestedTypeDeclarations;
+  }
+
+  void setHasNestedTypeDeclarations(bool value) {
+    Bits.AbstractFunctionDecl.HasNestedTypeDeclarations = value;
+  }
+
   /// Note that parsing for the body was delayed.
   ///
   /// The function should return the body statement and a flag indicating

include/swift/AST/DeclContext.h

@@ -873,8 +873,6 @@ class IterableDeclContext {
   /// available.
   Optional<std::string> getBodyFingerprint() const;
 
-  bool areTokensHashedForThisBodyInsteadOfInterfaceHash() const;
-
 private:
   /// Add a member to the list for iteration purposes, but do not notify the
   /// subclass that we have done so.

include/swift/AST/FineGrainedDependencies.h

@@ -351,7 +351,7 @@ class BiIndexedTwoStageMap {
 ///
 /// \Note The returned graph should not be escaped from the callback.
 bool withReferenceDependencies(
-    llvm::PointerUnion<ModuleDecl *, SourceFile *> MSF,
+    llvm::PointerUnion<const ModuleDecl *, const SourceFile *> MSF,
     const DependencyTracker &depTracker, StringRef outputPath,
     bool alsoEmitDotFile, llvm::function_ref<bool(SourceFileDepGraph &&)>);
 

include/swift/Basic/FunctionBodySkipping.h

@@ -23,6 +23,9 @@ enum class FunctionBodySkipping : uint8_t {
   None,
   /// Only non-inlinable function bodies should be skipped.
   NonInlinable,
+  /// Only non-inlinable functions bodies without type definitions should
+  /// be skipped.
+  NonInlinableWithoutTypes,
   /// All function bodies should be skipped, where not otherwise required
   /// for type inference.
   All

include/swift/Demangling/DemangleNodes.def

@@ -160,6 +160,7 @@ NODE(NominalTypeDescriptor)
 NODE(NonObjCAttribute)
 NODE(Number)
 NODE(ObjCAsyncCompletionHandlerImpl)
+NODE(PredefinedObjCAsyncCompletionHandlerImpl)
 NODE(ObjCAttribute)
 NODE(ObjCBlock)
 NODE(EscapingObjCBlock)

include/swift/Frontend/InputFile.h

@@ -121,17 +121,17 @@ class InputFile final {
   // FrontendInputsAndOutputs. They merely make the call sites
   // a bit shorter. Add more forwarding methods as needed.
 
-  std::string dependenciesFilePath() const {
+  StringRef getDependenciesFilePath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs.DependenciesFilePath;
   }
-  std::string loadedModuleTracePath() const {
+  StringRef getLoadedModuleTracePath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs.LoadedModuleTracePath;
   }
-  std::string serializedDiagnosticsPath() const {
+  StringRef getSerializedDiagnosticsPath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs
         .SerializedDiagnosticsPath;
   }
-  std::string fixItsOutputPath() const {
+  StringRef getFixItsOutputPath() const {
     return getPrimarySpecificPaths().SupplementaryOutputs.FixItsOutputPath;
   }
 };