Add explicit lowering for value packs and pack expansions.

- SILPackType carries whether the elements are stored directly
  in the pack, which we're not currently using in the lowering,
  but it's probably something we'll want in the final ABI.
  Having this also makes it clear that we're doing the right
  thing with substitution and element lowering.  I also toyed
  with making this a scalar type, which made it necessary in
  various places, although eventually I pulled back to the
  design where we always use packs as addresses.

- Pack boundaries are a core ABI concept, so the lowering has
  to wrap parameter pack expansions up as packs.  There are huge
  unimplemented holes here where the abstraction pattern will
  need to tell us how many elements to gather into the pack,
  but a naive approach is good enough to get things off the
  ground.

- Pack conventions are related to the existing parameter and
  result conventions, but they're different on enough grounds
  that they deserve to be separated.

Author: rjmccall

docs/ABI/Mangling.rst

@@ -665,6 +665,7 @@ Types
   
   type ::= pattern-type count-type 'Qp'      // pack expansion type
   type ::= pack-element-list 'QP'            // pack type
+  type ::= pack-element-list 'QS' DIRECTNESS // SIL pack type
 
   pack-element-list ::= type '_' type*
   pack-element-list ::= empty-list
@@ -746,6 +747,9 @@ mangled in to disambiguate.
   PARAM-CONVENTION ::= 'y'                   // direct unowned
   PARAM-CONVENTION ::= 'g'                   // direct guaranteed
   PARAM-CONVENTION ::= 'e'                   // direct deallocating
+  PARAM-CONVENTION ::= 'v'                   // pack owned
+  PARAM-CONVENTION ::= 'p'                   // pack guaranteed
+  PARAM-CONVENTION ::= 'm'                   // pack inout
 
   PARAM-DIFFERENTIABILITY ::= 'w'            // @noDerivative
 
@@ -754,9 +758,13 @@ mangled in to disambiguate.
   RESULT-CONVENTION ::= 'd'                  // unowned
   RESULT-CONVENTION ::= 'u'                  // unowned inner pointer
   RESULT-CONVENTION ::= 'a'                  // auto-released
+  RESULT-CONVENTION ::= 'k'                  // pack
 
   RESULT-DIFFERENTIABILITY ::= 'w'            // @noDerivative
 
+  DIRECTNESS ::= 'i'                         // indirect
+  DIRECTNESS ::= 'd'                         // direct
+
 For the most part, manglings follow the structure of formal language
 types.  However, in some cases it is more useful to encode the exact
 implementation details of a function type.

include/swift/AST/Attr.def.gyb

@@ -66,11 +66,16 @@ TYPE_ATTR(dynamic_self)
 #include "swift/AST/ReferenceStorage.def"
 TYPE_ATTR(error)
 TYPE_ATTR(out)
+TYPE_ATTR(direct)
 TYPE_ATTR(in)
 TYPE_ATTR(inout)
 TYPE_ATTR(inout_aliasable)
 TYPE_ATTR(in_guaranteed)
 TYPE_ATTR(in_constant)
+TYPE_ATTR(pack_owned)
+TYPE_ATTR(pack_guaranteed)
+TYPE_ATTR(pack_inout)
+TYPE_ATTR(pack_out)
 TYPE_ATTR(owned)
 TYPE_ATTR(unowned_inner_pointer)
 TYPE_ATTR(guaranteed)

include/swift/AST/TypeDifferenceVisitor.h

@@ -136,6 +136,14 @@ class CanTypeDifferenceVisitor : public CanTypePairVisitor<Impl, bool> {
                                type2->getElementTypes());
   }
 
+  bool visitSILPackType(CanSILPackType type1, CanSILPackType type2) {
+    if (type1->isElementAddress() != type2->isElementAddress())
+      return asImpl().visitDifferentTypeStructure(type1, type2);
+    return visitComponentArray(type1, type2,
+                               type1->getElementTypes(),
+                               type2->getElementTypes());
+  }
+
   bool visitPackExpansionType(CanPackExpansionType type1,
                               CanPackExpansionType type2) {
     return asImpl().visit(type1.getPatternType(), type2.getPatternType());

include/swift/AST/TypeMatcher.h

@@ -143,6 +143,29 @@ class TypeMatcher {
       return mismatch(firstTuple.getPointer(), secondType, sugaredFirstType);
     }
 
+    bool visitSILPackType(CanSILPackType firstPack, Type secondType,
+                          Type sugaredFirstType) {
+      if (auto secondPack = secondType->getAs<SILPackType>()) {
+        if (firstPack->getNumElements() != secondPack->getNumElements())
+          return mismatch(firstPack.getPointer(), secondPack,
+                          sugaredFirstType);
+
+        for (unsigned i = 0, n = firstPack->getNumElements(); i != n; ++i) {
+          // Recurse on the pack elements.  There's no need to preserve
+          // sugar for SIL types.
+          if (!this->visit(firstPack->getElementType(i),
+                           secondPack->getElementType(i),
+                           firstPack->getElementType(i)))
+            return false;
+        }
+
+        return true;
+      }
+
+      // Pack/non-pack mismatch.
+      return mismatch(firstPack.getPointer(), secondType, sugaredFirstType);
+    }
+
     bool visitPackType(CanPackType firstTuple, Type secondType,
                        Type sugaredFirstType) {
       if (auto secondTuple = secondType->getAs<PackType>()) {

include/swift/AST/TypeNodes.def

@@ -53,6 +53,17 @@
 ///   This is only expanded if SINGLETON_TYPE is defined, and in this
 ///   case no other macros are expanded.
 
+#ifndef ALWAYS_CANONICAL_ARTIFICIAL_TYPE
+#ifdef ALWAYS_CANONICAL_TYPE
+#ifdef ARTIFICIAL_TYPE
+#error must define ALWAYS_CANONICAL_ARTIFICIAL_TYPE if also defining both ALWAYS_CANONICAL_TYPE and ARTIFICIAL_TYPE
+#endif
+#define ALWAYS_CANONICAL_ARTIFICIAL_TYPE(id, parent) ALWAYS_CANONICAL_TYPE(id, parent)
+#else
+#define ALWAYS_CANONICAL_ARTIFICIAL_TYPE(id, parent) ARTIFICIAL_TYPE(id, parent)
+#endif
+#endif
+
 #ifndef ALWAYS_CANONICAL_TYPE
 #define ALWAYS_CANONICAL_TYPE(id, parent) TYPE(id, parent)
 #endif
@@ -165,6 +176,7 @@ ARTIFICIAL_TYPE(SILFunction, Type)
 ARTIFICIAL_TYPE(SILBlockStorage, Type)
 ARTIFICIAL_TYPE(SILBox, Type)
 ARTIFICIAL_TYPE(SILMoveOnlyWrapped, Type)
+ALWAYS_CANONICAL_ARTIFICIAL_TYPE(SILPack, Type)
 ARTIFICIAL_TYPE(SILToken, Type)
 TYPE(ProtocolComposition, Type)
 TYPE(ParameterizedProtocol, Type)
@@ -213,5 +225,6 @@ SINGLETON_TYPE(SILToken, SILToken)
 #undef SUGARED_TYPE
 #undef BUILTIN_TYPE
 #undef ALWAYS_CANONICAL_TYPE
+#undef ALWAYS_CANONICAL_ARTIFICIAL_TYPE
 #undef TYPE
 #undef LAST_TYPE

include/swift/AST/Types.h

@@ -407,10 +407,10 @@ class alignas(1 << TypeAlignInBits) TypeBase
     ID : 32
   );
 
-  SWIFT_INLINE_BITFIELD(SILFunctionType, TypeBase, NumSILExtInfoBits+1+3+1+2+1+1,
+  SWIFT_INLINE_BITFIELD(SILFunctionType, TypeBase, NumSILExtInfoBits+1+4+1+2+1+1,
     ExtInfoBits : NumSILExtInfoBits,
     HasClangTypeInfo : 1,
-    CalleeConvention : 3,
+    CalleeConvention : 4,
     HasErrorResult : 1,
     CoroutineKind : 2,
     HasInvocationSubs : 1,
@@ -451,7 +451,17 @@ class alignas(1 << TypeAlignInBits) TypeBase
   SWIFT_INLINE_BITFIELD_FULL(PackType, TypeBase, 32,
     : NumPadBits,
 
-    /// The number of elements of the tuple.
+    /// The number of elements of the pack.
+    Count : 32
+  );
+
+  SWIFT_INLINE_BITFIELD_FULL(SILPackType, TypeBase, 1+32,
+    /// Whether elements of the pack are addresses.
+    ElementIsAddress : 1,
+
+    : NumPadBits,
+
+    /// The number of elements of the pack
     Count : 32
   );
 
@@ -3777,9 +3787,24 @@ enum class ParameterConvention : uint8_t {
   /// This argument is passed directly.  Its type is non-trivial, and the caller
   /// guarantees its validity for the entirety of the call.
   Direct_Guaranteed,
+
+  /// This argument is a value pack of mutable references to storage,
+  /// which the function is being given exclusive access to.  The elements
+  /// must be passed indirectly.
+  Pack_Inout,
+
+  /// This argument is a value pack, and ownership of the elements is being
+  /// transferred into this function.  Whether the elements are passed
+  /// indirectly is recorded in the pack type.
+  Pack_Owned,
+
+  /// This argument is a value pack, and ownership of the elements is not
+  /// being transferred into this function.  Whether the elements are passed
+  /// indirectly is recorded in the pack type.
+  Pack_Guaranteed,
 };
 // Check that the enum values fit inside Bits.SILFunctionType.
-static_assert(unsigned(ParameterConvention::Direct_Guaranteed) < (1<<3),
+static_assert(unsigned(ParameterConvention::Pack_Guaranteed) < (1<<4),
               "fits in Bits.SILFunctionType");
 
 // Does this parameter convention require indirect storage? This reflects a
@@ -3796,6 +3821,9 @@ inline bool isIndirectFormalParameter(ParameterConvention conv) {
   case ParameterConvention::Direct_Unowned:
   case ParameterConvention::Direct_Guaranteed:
   case ParameterConvention::Direct_Owned:
+  case ParameterConvention::Pack_Inout:
+  case ParameterConvention::Pack_Owned:
+  case ParameterConvention::Pack_Guaranteed:
     return false;
   }
   llvm_unreachable("covered switch isn't covered?!");
@@ -3804,13 +3832,16 @@ inline bool isConsumedParameter(ParameterConvention conv) {
   switch (conv) {
   case ParameterConvention::Indirect_In:
   case ParameterConvention::Direct_Owned:
+  case ParameterConvention::Pack_Owned:
     return true;
 
   case ParameterConvention::Indirect_Inout:
   case ParameterConvention::Indirect_InoutAliasable:
   case ParameterConvention::Direct_Unowned:
   case ParameterConvention::Direct_Guaranteed:
   case ParameterConvention::Indirect_In_Guaranteed:
+  case ParameterConvention::Pack_Inout:
+  case ParameterConvention::Pack_Guaranteed:
     return false;
   }
   llvm_unreachable("bad convention kind");
@@ -3823,11 +3854,34 @@ inline bool isGuaranteedParameter(ParameterConvention conv) {
   switch (conv) {
   case ParameterConvention::Direct_Guaranteed:
   case ParameterConvention::Indirect_In_Guaranteed:
+  case ParameterConvention::Pack_Guaranteed:
+    return true;
+
+  case ParameterConvention::Indirect_Inout:
+  case ParameterConvention::Indirect_InoutAliasable:
+  case ParameterConvention::Indirect_In:
+  case ParameterConvention::Direct_Unowned:
+  case ParameterConvention::Direct_Owned:
+  case ParameterConvention::Pack_Inout:
+  case ParameterConvention::Pack_Owned:
+    return false;
+  }
+  llvm_unreachable("bad convention kind");
+}
+
+/// Returns true if conv indicates a pack parameter.
+inline bool isPackParameter(ParameterConvention conv) {
+  switch (conv) {
+  case ParameterConvention::Pack_Guaranteed:
+  case ParameterConvention::Pack_Inout:
+  case ParameterConvention::Pack_Owned:
     return true;
 
+  case ParameterConvention::Indirect_In_Guaranteed:
   case ParameterConvention::Indirect_Inout:
   case ParameterConvention::Indirect_InoutAliasable:
   case ParameterConvention::Indirect_In:
+  case ParameterConvention::Direct_Guaranteed:
   case ParameterConvention::Direct_Unowned:
   case ParameterConvention::Direct_Owned:
     return false;
@@ -3906,6 +3960,10 @@ class SILParameterInfo {
         || getConvention() == ParameterConvention::Indirect_InoutAliasable;
   }
 
+  bool isPack() const {
+    return isPackParameter(getConvention());
+  }
+
   /// True if this parameter is consumed by the callee, either
   /// indirectly or directly.
   bool isConsumed() const {
@@ -4027,11 +4085,18 @@ enum class ResultConvention : uint8_t {
   /// The type must be a class or class existential type, and this
   /// must be the only return value.
   Autoreleased,
+
+  /// This value is a pack that is returned indirectly by passing a
+  /// pack address (which may or may not be further indirected,
+  /// depending on the pact type).  The callee is responsible for
+  /// leaving an initialized object in each element of the pack.
+  Pack,
 };
 
 // Does this result require indirect storage for the purpose of reabstraction?
 inline bool isIndirectFormalResult(ResultConvention convention) {
-  return convention == ResultConvention::Indirect;
+  return convention == ResultConvention::Indirect ||
+         convention == ResultConvention::Pack;
 }
 
 /// The differentiability of a SIL function type result.
@@ -5138,6 +5203,97 @@ class SILTokenType final : public TypeBase {
 };
 DEFINE_EMPTY_CAN_TYPE_WRAPPER(SILTokenType, Type)
 
+/// A lowered pack type which structurally carries lowered information
+/// about the pack elements.
+///
+/// A value pack is basically treated as a unique-ownership, unmovable
+/// reference-semantics aggregate in SIL: ownership of the pack as a whole
+/// is communicated with normal borrows of the pack, and packs can only
+/// be created locally and forwarded as arguments rather than being moved
+/// in any more complex way.
+class SILPackType final : public TypeBase, public llvm::FoldingSetNode,
+    private llvm::TrailingObjects<SILPackType, CanType> {
+public:
+  /// Type structure not reflected in the pack element type list.
+  ///
+  /// In the design of this, we considered storing ownership here,
+  /// but ended up just with the one bit.
+  struct ExtInfo {
+    bool ElementIsAddress;
+
+    ExtInfo(bool elementIsAddress) : ElementIsAddress(elementIsAddress) {}
+  };
+
+private:
+  friend TrailingObjects;
+  friend class ASTContext;
+  SILPackType(const ASTContext &ctx, RecursiveTypeProperties properties,
+              ExtInfo info, ArrayRef<CanType> elements)
+      : TypeBase(TypeKind::SILPack, &ctx, properties) {
+    Bits.SILPackType.Count = elements.size();
+    Bits.SILPackType.ElementIsAddress = info.ElementIsAddress;
+    memcpy(getTrailingObjects<CanType>(), elements.data(),
+           elements.size() * sizeof(CanType));
+  }
+
+public:
+  static CanTypeWrapper<SILPackType> get(const ASTContext &ctx,
+                                         ExtInfo info,
+                                         ArrayRef<CanType> elements);
+
+  ExtInfo getExtInfo() const {
+    return { isElementAddress() };
+  }
+
+  bool isElementAddress() const {
+    return Bits.SILPackType.ElementIsAddress;
+  }
+
+  /// Retrieves the number of elements in this pack.
+  unsigned getNumElements() const { return Bits.SILPackType.Count; }
+
+  /// Retrieves the type of the elements in the pack.
+  ArrayRef<CanType> getElementTypes() const {
+    return {getTrailingObjects<CanType>(), getNumElements()};
+  }
+
+  /// Returns the type of the element at the given \p index.
+  /// This is a lowered SIL type.
+  CanType getElementType(unsigned index) const {
+    return getTrailingObjects<CanType>()[index];
+  }
+
+  SILType getSILElementType(unsigned index) const; // in SILType.h
+
+  /// Return the reduced shape of this pack.  For consistency with
+  /// general shape-handling routines, we produce an AST pack type
+  /// as the shape, not a SIL pack type.
+  CanTypeWrapper<PackType> getReducedShape() const;
+
+  bool containsPackExpansionType() const;
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    Profile(ID, getExtInfo(), getElementTypes());
+  }
+  static void Profile(llvm::FoldingSetNodeID &ID,
+                      ExtInfo info,
+                      ArrayRef<CanType> elements);
+
+  // Implement isa/cast/dyncast/etc.
+  static bool classof(const TypeBase *T) {
+    return T->getKind() == TypeKind::SILPack;
+  }
+};
+BEGIN_CAN_TYPE_WRAPPER(SILPackType, Type)
+  CanType getElementType(unsigned elementNo) const {
+    return getPointer()->getElementType(elementNo);
+  }
+
+  ArrayRef<CanType> getElementTypes() const {
+    return getPointer()->getElementTypes();
+  }
+END_CAN_TYPE_WRAPPER(SILPackType, Type)
+
 /// A type with a special syntax that is always sugar for a library type. The
 /// library type may have multiple base types. For unary syntax sugar, see
 /// UnarySyntaxSugarType.
@@ -6654,6 +6810,8 @@ class PackExpansionType : public TypeBase, public llvm::FoldingSetNode {
                     const ASTContext *ctx);
 };
 BEGIN_CAN_TYPE_WRAPPER(PackExpansionType, Type)
+  static CanPackExpansionType get(CanType pattern, CanType countType);
+
   CanType getPatternType() const {
     return CanType(getPointer()->getPatternType());
   }