stdlib: some small Array improvements to prepare for COW representation

stdlib/public/core/Array.swift

@@ -346,8 +346,7 @@ extension Array {
   @_semantics("array.make_mutable")
   internal mutating func _makeMutableAndUnique() {
     if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
-      _createNewBuffer(bufferIsUnique: false, minimumCapacity: count,
-                       growForAppend: false)
+      _buffer = _buffer._consumeAndCreateNew()
     }
   }
 
@@ -1049,34 +1048,13 @@ extension Array: RangeReplaceableCollection {
   /// If `growForAppend` is true, the new capacity is calculated using
   /// `_growArrayCapacity`, but at least kept at `minimumCapacity`.
   @_alwaysEmitIntoClient
-  @inline(never)
   internal mutating func _createNewBuffer(
     bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
   ) {
-    let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(),
-                                         minimumCapacity: minimumCapacity,
-                                         growForAppend: growForAppend)
-    let count = _getCount()
-    _internalInvariant(newCapacity >= count)
-
-    let newBuffer = _ContiguousArrayBuffer<Element>(
-      _uninitializedCount: count, minimumCapacity: newCapacity)
-
-    if bufferIsUnique {
-      _internalInvariant(_buffer.isUniquelyReferenced())
-
-      // As an optimization, if the original buffer is unique, we can just move
-      // the elements instead of copying.
-      let dest = newBuffer.firstElementAddress
-      dest.moveInitialize(from: _buffer.firstElementAddress,
-                          count: count)
-      _buffer.count = 0
-    } else {
-      _buffer._copyContents(
-        subRange: 0..<count,
-        initializing: newBuffer.firstElementAddress)
-    }
-    _buffer = _Buffer(_buffer: newBuffer, shiftedToStartIndex: 0)
+    _internalInvariant(!bufferIsUnique || _buffer.isUniquelyReferenced())
+    _buffer = _buffer._consumeAndCreateNew(bufferIsUnique: bufferIsUnique,
+                                           minimumCapacity: minimumCapacity,
+                                           growForAppend: growForAppend)
   }
 
   /// Copy the contents of the current buffer to a new unique mutable buffer.

stdlib/public/core/ArrayBuffer.swift

@@ -125,6 +125,60 @@ extension _ArrayBuffer {
     return _fastPath(_isNative) ? _native._asCocoaArray() : _nonNative.buffer
   }
 
+  /// Creates and returns a new uniquely referenced buffer which is a copy of
+  /// this buffer.
+  ///
+  /// This buffer is consumed, i.e. it's released.
+  @_alwaysEmitIntoClient
+  @inline(never)
+  @_semantics("optimize.sil.specialize.owned2guarantee.never")
+  internal __consuming func _consumeAndCreateNew() -> _ArrayBuffer {
+    return _consumeAndCreateNew(bufferIsUnique: false,
+                                minimumCapacity: count,
+                                growForAppend: false)
+  }
+
+  /// Creates and returns a new uniquely referenced buffer which is a copy of
+  /// this buffer.
+  ///
+  /// If `bufferIsUnique` is true, the buffer is assumed to be uniquely
+  /// referenced and the elements are moved - instead of copied - to the new
+  /// buffer.
+  /// The `minimumCapacity` is the lower bound for the new capacity.
+  /// If `growForAppend` is true, the new capacity is calculated using
+  /// `_growArrayCapacity`, but at least kept at `minimumCapacity`.
+  ///
+  /// This buffer is consumed, i.e. it's released.
+  @_alwaysEmitIntoClient
+  @inline(never)
+  @_semantics("optimize.sil.specialize.owned2guarantee.never")
+  internal __consuming func _consumeAndCreateNew(
+    bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
+  ) -> _ArrayBuffer {
+    let newCapacity = _growArrayCapacity(oldCapacity: capacity,
+                                         minimumCapacity: minimumCapacity,
+                                         growForAppend: growForAppend)
+    let c = count
+    _internalInvariant(newCapacity >= c)
+
+    let newBuffer = _ContiguousArrayBuffer<Element>(
+      _uninitializedCount: c, minimumCapacity: newCapacity)
+
+    if bufferIsUnique {
+      // As an optimization, if the original buffer is unique, we can just move
+      // the elements instead of copying.
+      let dest = newBuffer.firstElementAddress
+      dest.moveInitialize(from: firstElementAddress,
+                          count: c)
+      _native.count = 0
+    } else {
+      _copyContents(
+        subRange: 0..<c,
+        initializing: newBuffer.firstElementAddress)
+    }
+    return _ArrayBuffer(_buffer: newBuffer, shiftedToStartIndex: 0)
+  }
+
   /// If this buffer is backed by a uniquely-referenced mutable
   /// `_ContiguousArrayBuffer` that can be grown in-place to allow the self
   /// buffer store minimumCapacity elements, returns that buffer.

stdlib/public/core/ArrayBufferProtocol.swift

@@ -150,7 +150,10 @@ extension _ArrayBufferProtocol where Indices == Range<Int>{
     let eraseCount = subrange.count
 
     let growth = newCount - eraseCount
-    self.count = oldCount + growth
+    // This check will prevent storing a 0 count to the empty array singleton.
+    if growth != 0 {
+      self.count = oldCount + growth
+    }
 
     let elements = self.subscriptBaseAddress
     let oldTailIndex = subrange.upperBound

stdlib/public/core/ContiguousArray.swift

@@ -67,8 +67,7 @@ extension ContiguousArray {
   @_semantics("array.make_mutable")
   internal mutating func _makeMutableAndUnique() {
     if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
-      _createNewBuffer(bufferIsUnique: false, minimumCapacity: count,
-                       growForAppend: false)
+      _buffer = _buffer._consumeAndCreateNew()
     }
   }
 
@@ -690,30 +689,10 @@ extension ContiguousArray: RangeReplaceableCollection {
   internal mutating func _createNewBuffer(
     bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
   ) {
-    let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(),
-                                         minimumCapacity: minimumCapacity,
-                                         growForAppend: growForAppend)
-    let count = _getCount()
-    _internalInvariant(newCapacity >= count)
-
-    let newBuffer = _ContiguousArrayBuffer<Element>(
-      _uninitializedCount: count, minimumCapacity: newCapacity)
-
-    if bufferIsUnique {
-      _internalInvariant(_buffer.isUniquelyReferenced())
-
-      // As an optimization, if the original buffer is unique, we can just move
-      // the elements instead of copying.
-      let dest = newBuffer.firstElementAddress
-      dest.moveInitialize(from: _buffer.firstElementAddress,
-                          count: count)
-      _buffer.count = 0
-    } else {
-      _buffer._copyContents(
-        subRange: 0..<count,
-        initializing: newBuffer.firstElementAddress)
-    }
-    _buffer = _Buffer(_buffer: newBuffer, shiftedToStartIndex: 0)
+    _internalInvariant(!bufferIsUnique || _buffer.isUniquelyReferenced())
+    _buffer = _buffer._consumeAndCreateNew(bufferIsUnique: bufferIsUnique,
+                                           minimumCapacity: minimumCapacity,
+                                           growForAppend: growForAppend)
   }
 
   /// Copy the contents of the current buffer to a new unique mutable buffer.

stdlib/public/core/ContiguousArrayBuffer.swift

@@ -511,6 +511,60 @@ internal struct _ContiguousArrayBuffer<Element>: _ArrayBufferProtocol {
     return _isUnique(&_storage)
   }
 
+  /// Creates and returns a new uniquely referenced buffer which is a copy of
+  /// this buffer.
+  ///
+  /// This buffer is consumed, i.e. it's released.
+  @_alwaysEmitIntoClient
+  @inline(never)
+  @_semantics("optimize.sil.specialize.owned2guarantee.never")
+  internal __consuming func _consumeAndCreateNew() -> _ContiguousArrayBuffer {
+    return _consumeAndCreateNew(bufferIsUnique: false,
+                                minimumCapacity: count,
+                                growForAppend: false)
+  }
+
+  /// Creates and returns a new uniquely referenced buffer which is a copy of
+  /// this buffer.
+  ///
+  /// If `bufferIsUnique` is true, the buffer is assumed to be uniquely
+  /// referenced and the elements are moved - instead of copied - to the new
+  /// buffer.
+  /// The `minimumCapacity` is the lower bound for the new capacity.
+  /// If `growForAppend` is true, the new capacity is calculated using
+  /// `_growArrayCapacity`, but at least kept at `minimumCapacity`.
+  ///
+  /// This buffer is consumed, i.e. it's released.
+  @_alwaysEmitIntoClient
+  @inline(never)
+  @_semantics("optimize.sil.specialize.owned2guarantee.never")
+  internal __consuming func _consumeAndCreateNew(
+    bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
+  ) -> _ContiguousArrayBuffer {
+    let newCapacity = _growArrayCapacity(oldCapacity: capacity,
+                                         minimumCapacity: minimumCapacity,
+                                         growForAppend: growForAppend)
+    let c = count
+    _internalInvariant(newCapacity >= c)
+
+    let newBuffer = _ContiguousArrayBuffer<Element>(
+      _uninitializedCount: c, minimumCapacity: newCapacity)
+
+    if bufferIsUnique {
+      // As an optimization, if the original buffer is unique, we can just move
+      // the elements instead of copying.
+      let dest = newBuffer.firstElementAddress
+      dest.moveInitialize(from: firstElementAddress,
+                          count: c)
+      count = 0
+    } else {
+      _copyContents(
+        subRange: 0..<c,
+        initializing: newBuffer.firstElementAddress)
+    }
+    return newBuffer
+  }
+
 #if _runtime(_ObjC)
 
   /// Convert to an NSArray.

stdlib/public/core/SetAlgebra.swift

@@ -409,6 +409,8 @@ extension SetAlgebra {
   public init<S: Sequence>(_ sequence: __owned S)
     where S.Element == Element {
     self.init()
+    // Needed to fully optimize OptionSet literals.
+    _onFastPath()
     for e in sequence { insert(e) }
   }
 

test/IRGen/multithread_module.swift

@@ -68,8 +68,8 @@ public func mutateBaseArray(_ arr: inout [Base], _ x: Base) {
 
 // Check if all specializations from stdlib functions are created in the same LLVM module.
 
-// CHECK-MAINLL-DAG: define {{.*}} @"$sSa16_createNewBuffer14bufferIsUnique15minimumCapacity13growForAppendySb_SiSbtF4test8MyStructV_Tg5"
-// CHECK-MAINLL-DAG: define {{.*}} @"$sSa16_createNewBuffer14bufferIsUnique15minimumCapacity13growForAppendySb_SiSbtF4test4BaseC_Tg5"
+// CHECK-MAINLL-DAG: define {{.*}} @"$ss{{(12_|22_Contiguous)}}ArrayBufferV20_consumeAndCreateNew14bufferIsUnique15minimumCapacity13growForAppendAByxGSb_SiSbtF4test8MyStructV_Tg5"
+// CHECK-MAINLL-DAG: define {{.*}} @"$ss{{(12_|22_Contiguous)}}ArrayBufferV20_consumeAndCreateNew14bufferIsUnique15minimumCapacity13growForAppendAByxGSb_SiSbtF4test4BaseC_Tg5"
 
 // Check if the DI filename is correct and not "<unknown>".
 

test/SILOptimizer/array_contentof_opt.swift

@@ -1,4 +1,4 @@
-// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil -Xllvm '-sil-inline-never-functions=$sSa6append' %s | %FileCheck %s
+// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil -Xllvm '-sil-inline-never-functions=$sSa6appendyy' %s | %FileCheck %s
 // REQUIRES: swift_stdlib_no_asserts,optimized_stdlib
 
 // This is an end-to-end test of the Array.append(contentsOf:) ->
@@ -24,7 +24,7 @@ public func testInt(_ a: inout [Int]) {
 }
 
 // CHECK-LABEL: sil @{{.*}}testThreeInts
-// CHECK-DAG:    [[FR:%[0-9]+]] = function_ref @${{(sSa15reserveCapacityyySiFSi_Tg5|sSa16_createNewBuffer)}}
+// CHECK-DAG:    [[FR:%[0-9]+]] = function_ref @${{.*(reserveCapacity|_createNewBuffer)}}
 // CHECK-DAG:    apply [[FR]]
 // CHECK-DAG:    [[F:%[0-9]+]] = function_ref @$sSa6appendyyxnFSi_Tg5
 // CHECK-DAG:    apply [[F]]
@@ -37,7 +37,7 @@ public func testThreeInts(_ a: inout [Int]) {
 
 // CHECK-LABEL: sil @{{.*}}testTooManyInts
 // CHECK-NOT: apply
-// CHECK:        [[F:%[0-9]+]] = function_ref  @$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5
+// CHECK:        [[F:%[0-9]+]] = function_ref  @${{.*append.*contentsOf.*}}
 // CHECK-NOT: apply
 // CHECK:        apply [[F]]
 // CHECK-NOT: apply
@@ -65,12 +65,12 @@ public func dontPropagateContiguousArray(_ a: inout ContiguousArray<UInt8>) {
 
 // Check if the specialized Array.append<A>(contentsOf:) is reasonably optimized for Array<Int>.
 
-// CHECK-LABEL: sil shared {{.*}}@$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5Tf4gn_n
+// CHECK-LABEL: sil shared {{.*}}@$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5
 
 // There should only be a single call to _createNewBuffer or reserveCapacityForAppend/reserveCapacityImpl.
 
 // CHECK-NOT: apply
-// CHECK: [[F:%[0-9]+]] = function_ref @{{.*(_createNewBuffer|reserveCapacity).*}}
+// CHECK: [[F:%[0-9]+]] = function_ref @{{.*(_consumeAndCreateNew|reserveCapacity).*}}
 // CHECK-NEXT: apply [[F]]
 // CHECK-NOT: apply
 

test/SILOptimizer/optionset.swift

@@ -14,6 +14,7 @@ public struct TestOptions: OptionSet {
 
 // CHECK:      sil @{{.*}}returnTestOptions{{.*}}
 // CHECK-NEXT: bb0:
+// CHECK-NEXT:   builtin
 // CHECK-NEXT:   integer_literal {{.*}}, 15
 // CHECK-NEXT:   struct $Int
 // CHECK-NEXT:   struct $TestOptions
@@ -22,18 +23,19 @@ public func returnTestOptions() -> TestOptions {
     return [.first, .second, .third, .fourth]
 }
 
-// CHECK:      sil @{{.*}}returnEmptyTestOptions{{.*}}
-// CHECK-NEXT: bb0:
-// CHECK-NEXT:   integer_literal {{.*}}, 0
-// CHECK-NEXT:   struct $Int
-// CHECK-NEXT:   struct $TestOptions
-// CHECK-NEXT:   return
+// CHECK: sil @{{.*}}returnEmptyTestOptions{{.*}}
+// CHECK:   [[ZERO:%[0-9]+]] = integer_literal {{.*}}, 0
+// CHECK:   [[ZEROINT:%[0-9]+]] = struct $Int ([[ZERO]]
+// CHECK:   [[TO:%[0-9]+]] = struct $TestOptions ([[ZEROINT]]
+// CHECK:   return [[TO]]
+// CHECK: } // end sil function {{.*}}returnEmptyTestOptions{{.*}}
 public func returnEmptyTestOptions() -> TestOptions {
     return []
 }
 
 // CHECK:        alloc_global @{{.*}}globalTestOptions{{.*}}
 // CHECK-NEXT:   global_addr
+// CHECK-NEXT:   builtin
 // CHECK-NEXT:   integer_literal {{.*}}, 15
 // CHECK-NEXT:   struct $Int
 // CHECK-NEXT:   struct $TestOptions