[4.2][stdlib] Fix AnyHashable's Equatable/Hashable conformance (#17518)

* [stdlib] Fix AnyHashable's Equatable/Hashable conformance

AnyHashable has numerous edge cases where two AnyHashable values compare equal but produce different hashes. This breaks Set and Dictionary invariants and can cause unexpected behavior and/or traps. This change overhauls AnyHashable's implementation to fix these edge cases, hopefully without introducing new issues.

- Fix transitivity of ==. Previously, comparisons involving AnyHashable values with Objective-C provenance were handled specially, breaking Equatable:

    let a = (42 as Int as AnyHashable)
    let b = (42 as NSNumber as AnyHashable)
    let c = (42 as Double as AnyHashable)
    a == b // true
    b == c // true
    a == c // was false(!), now true

    let d = ("foo" as AnyHashable)
    let e = ("foo" as NSString as AnyHashable)
    let f = ("foo" as NSString as NSAttributedStringKey as AnyHashable)
    d == e // true
    e == f // true
    d == f // was false(!), now true

- Fix Hashable conformance for numeric types boxed into AnyHashable:

    b == c // true
    b.hashValue == c.hashValue // was false(!), now true

  Fixing this required adding a custom AnyHashable box for all standard integer and floating point types. The custom box was needed to ensure that two AnyHashables containing the same number compare equal and hash the same way, no matter what their original type was. (This behavior is required to ensure consistency with NSNumber, which has not been preserving types since SE-0170.

- Add custom AnyHashable representations for Arrays, Sets and Dictionaries, so that when they contain numeric types, they hash correctly under the new rules above.

- Remove AnyHashable._usedCustomRepresentation. The provenance of a value should not affect its behavior.

- Allow AnyHashable values to be downcasted into compatible types more often.

- Forward _rawHashValue(seed:) to AnyHashable box. This fixes AnyHashable hashing for types that customize single-shot hashing.

https://bugs.swift.org/browse/SR-7496
rdar://problem/39648819
(cherry picked from commit ff91f36)

# Conflicts:
#	stdlib/public/core/AnyHashable.swift
#	stdlib/public/core/Arrays.swift.gyb
#	stdlib/public/core/Hasher.swift
#	stdlib/public/core/Integers.swift.gyb

* [stdlib][SE-0206] Use distinct hash encodings for standard integer types

Fix Hashable conformance of standard integer types so that the number of bits they feed into hasher is exactly Self.bitWidth.

This was intended to be part of SE-0206. However, it would have introduced additional issues with AnyHashable. The custom AnyHashable representations introduced in the previous commit unify hashing for numeric types, eliminating the problem.

(cherry picked from commit bf872ec)

# Conflicts:
#	stdlib/public/core/Integers.swift.gyb

* [stdlib] Eliminate _AnyHashableBox._asSet() & ._asDictionary()

_canonicalBox can perform essentially the same job, so there is no reason to have these requirements.

(cherry picked from commit 2f4ad79)

Author: lorentey

stdlib/private/StdlibUnittest/StdlibUnittest.swift

@@ -2410,6 +2410,33 @@ internal func hash<H: Hashable>(_ value: H, seed: Int? = nil) -> Int {
   return hasher.finalize()
 }
 
+/// Test that the elements of `groups` consist of instances that satisfy the
+/// semantic requirements of `Hashable`, with each group defining a distinct
+/// equivalence class under `==`.
+public func checkHashableGroups<Groups: Collection>(
+  _ groups: Groups,
+  _ message: @autoclosure () -> String = "",
+  stackTrace: SourceLocStack = SourceLocStack(),
+  showFrame: Bool = true,
+  file: String = #file, line: UInt = #line
+) where Groups.Element: Collection, Groups.Element.Element: Hashable {
+  let instances = groups.flatMap { $0 }
+  // groupIndices[i] is the index of the element in groups that contains
+  // instances[i].
+  let groupIndices =
+    zip(0..., groups).flatMap { i, group in group.map { _ in i } }
+  func equalityOracle(_ lhs: Int, _ rhs: Int) -> Bool {
+    return groupIndices[lhs] == groupIndices[rhs]
+  }
+  checkHashable(
+    instances,
+    equalityOracle: equalityOracle,
+    hashEqualityOracle: equalityOracle,
+    allowBrokenTransitivity: false,
+    stackTrace: stackTrace.pushIf(showFrame, file: file, line: line),
+    showFrame: false)
+}
+
 /// Test that the elements of `instances` satisfy the semantic requirements of
 /// `Hashable`, using `equalityOracle` to generate equality and hashing
 /// expectations from pairs of positions in `instances`.

stdlib/public/core/AnyHashable.swift

@@ -39,21 +39,28 @@ public protocol _HasCustomAnyHashableRepresentation {
 
 @usableFromInline // FIXME(sil-serialize-all)
 internal protocol _AnyHashableBox {
-  var _typeID: ObjectIdentifier { get }
-  func _unbox<T : Hashable>() -> T?
+  var _canonicalBox: _AnyHashableBox { get }
 
   /// Determine whether values in the boxes are equivalent.
   ///
+  /// - Precondition: `self` and `box` are in canonical form.
   /// - Returns: `nil` to indicate that the boxes store different types, so
   ///   no comparison is possible. Otherwise, contains the result of `==`.
-  func _isEqual(to: _AnyHashableBox) -> Bool?
+  func _isEqual(to box: _AnyHashableBox) -> Bool?
   var _hashValue: Int { get }
   func _hash(into hasher: inout Hasher)
 
   var _base: Any { get }
+  func _unbox<T: Hashable>() -> T?
   func _downCastConditional<T>(into result: UnsafeMutablePointer<T>) -> Bool
 }
 
+extension _AnyHashableBox {
+  var _canonicalBox: _AnyHashableBox {
+    return self
+  }
+}
+
 @_fixed_layout // FIXME(sil-serialize-all)
 @usableFromInline // FIXME(sil-serialize-all)
 internal struct _ConcreteHashableBox<Base : Hashable> : _AnyHashableBox {
@@ -108,19 +115,6 @@ internal struct _ConcreteHashableBox<Base : Hashable> : _AnyHashableBox {
   }
 }
 
-#if _runtime(_ObjC)
-// Retrieve the custom AnyHashable representation of the value after it
-// has been bridged to Objective-C. This mapping to Objective-C and back
-// turns a non-custom representation into a custom one, which is used as
-// the lowest-common-denominator for comparisons.
-@inlinable // FIXME(sil-serialize-all)
-internal func _getBridgedCustomAnyHashable<T>(_ value: T) -> AnyHashable? {
-  let bridgedValue = _bridgeAnythingToObjectiveC(value)
-  return (bridgedValue as?
-    _HasCustomAnyHashableRepresentation)?._toCustomAnyHashable()
-}
-#endif
-
 /// A type-erased hashable value.
 ///
 /// The `AnyHashable` type forwards equality comparisons and hashing operations
@@ -144,8 +138,11 @@ internal func _getBridgedCustomAnyHashable<T>(_ value: T) -> AnyHashable? {
 public struct AnyHashable {
   @usableFromInline // FIXME(sil-serialize-all)
   internal var _box: _AnyHashableBox
-  @usableFromInline // FIXME(sil-serialize-all)
-  internal var _usedCustomRepresentation: Bool
+
+  @inlinable // FIXME(sil-serialize-all)
+  internal init(_box box: _AnyHashableBox) {
+    self._box = box
+  }
 
   /// Creates a type-erased hashable value that wraps the given instance.
   ///
@@ -167,15 +164,13 @@ public struct AnyHashable {
   /// - Parameter base: A hashable value to wrap.
   @inlinable // FIXME(sil-serialize-all)
   public init<H : Hashable>(_ base: H) {
-    if let customRepresentation =
+    if let custom =
       (base as? _HasCustomAnyHashableRepresentation)?._toCustomAnyHashable() {
-      self = customRepresentation
-      self._usedCustomRepresentation = true
+      self = custom
       return
     }
 
-    self._box = _ConcreteHashableBox(0 as Int)
-    self._usedCustomRepresentation = false
+    self.init(_box: _ConcreteHashableBox(false)) // Dummy value
     _makeAnyHashableUpcastingToHashableBaseType(
       base,
       storingResultInto: &self)
@@ -184,7 +179,6 @@ public struct AnyHashable {
   @inlinable // FIXME(sil-serialize-all)
   internal init<H : Hashable>(_usingDefaultRepresentationOf base: H) {
     self._box = _ConcreteHashableBox(base)
-    self._usedCustomRepresentation = false
   }
 
   /// The value wrapped by this instance.
@@ -213,13 +207,11 @@ public struct AnyHashable {
     if _box._downCastConditional(into: result) { return true }
 
     #if _runtime(_ObjC)
-    // If we used a custom representation, bridge to Objective-C and then
-    // attempt the cast from there.
-    if _usedCustomRepresentation {
-      if let value = _bridgeAnythingToObjectiveC(_box._base) as? T {
-        result.initialize(to: value)
-        return true
-      }
+    // Bridge to Objective-C and then attempt the cast from there.
+    // FIXME: This should also work without the Objective-C runtime.
+    if let value = _bridgeAnythingToObjectiveC(_box._base) as? T {
+      result.initialize(to: value)
+      return true
     }
     #endif
 
@@ -255,42 +247,15 @@ extension AnyHashable : Equatable {
   ///   - rhs: Another type-erased hashable value.
   @inlinable // FIXME(sil-serialize-all)
   public static func == (lhs: AnyHashable, rhs: AnyHashable) -> Bool {
-    // If they're equal, we're done.
-    if let result = lhs._box._isEqual(to: rhs._box) { return result }
-
-    #if _runtime(_ObjC)
-    // If one used a custom representation but the other did not, bridge
-    // the one that did *not* use the custom representation to Objective-C:
-    // if the bridged result has a custom representation, compare those custom
-    // custom representations.
-    if lhs._usedCustomRepresentation != rhs._usedCustomRepresentation {
-      // If the lhs used a custom representation, try comparing against the
-      // custom representation of the bridged rhs (if there is one).
-      if lhs._usedCustomRepresentation {
-        if let customRHS = _getBridgedCustomAnyHashable(rhs._box._base) {
-          return lhs._box._isEqual(to: customRHS._box) ?? false
-        }
-        return false
-      }
-
-      // Otherwise, try comparing the rhs against the custom representation of
-      // the bridged lhs (if there is one).
-      if let customLHS = _getBridgedCustomAnyHashable(lhs._box._base) {
-        return customLHS._box._isEqual(to: rhs._box) ?? false
-      }
-      return false
-    }
-    #endif
-
-    return false
+    return lhs._box._canonicalBox._isEqual(to: rhs._box._canonicalBox) ?? false
   }
 }
 
 extension AnyHashable : Hashable {
   /// The hash value.
   @inlinable // FIXME(sil-serialize-all)
   public var hashValue: Int {
-    return _box._hashValue
+    return _box._canonicalBox._hashValue
   }
 
   /// Hashes the essential components of this value by feeding them into the
@@ -300,7 +265,7 @@ extension AnyHashable : Hashable {
   ///   of this instance.
   @inlinable // FIXME(sil-serialize-all)
   public func hash(into hasher: inout Hasher) {
-    _box._hash(into: &hasher)
+    _box._canonicalBox._hash(into: &hasher)
   }
 }
 

stdlib/public/core/Arrays.swift.gyb

@@ -2420,6 +2420,73 @@ extension ArraySlice {
   }
 }
 
+extension Array: _HasCustomAnyHashableRepresentation
+  where Element: Hashable {
+  public func _toCustomAnyHashable() -> AnyHashable? {
+    return AnyHashable(_box: _ArrayAnyHashableBox(self))
+  }
+}
+
+internal protocol _ArrayAnyHashableProtocol: _AnyHashableBox {
+  var count: Int { get }
+  subscript(index: Int) -> AnyHashable { get }
+}
+
+internal struct _ArrayAnyHashableBox<Element: Hashable>
+  : _ArrayAnyHashableProtocol {
+  internal let _value: [Element]
+
+  internal init(_ value: [Element]) {
+    self._value = value
+  }
+
+  internal var _base: Any {
+    return _value
+  }
+
+  internal var count: Int {
+    return _value.count
+  }
+
+  internal subscript(index: Int) -> AnyHashable {
+    return _value[index] as AnyHashable
+  }
+
+  func _isEqual(to other: _AnyHashableBox) -> Bool? {
+    guard let other = other as? _ArrayAnyHashableProtocol else { return nil }
+    guard _value.count == other.count else { return false }
+    for i in 0 ..< _value.count {
+      if self[i] != other[i] { return false }
+    }
+    return true
+  }
+
+  var _hashValue: Int {
+    var hasher = Hasher()
+    _hash(into: &hasher)
+    return hasher.finalize()
+  }
+
+  func _hash(into hasher: inout Hasher) {
+    hasher.combine(_value.count) // discriminator
+    for i in 0 ..< _value.count {
+      hasher.combine(self[i])
+    }
+  }
+
+  internal func _unbox<T : Hashable>() -> T? {
+    return _value as? T
+  }
+
+  internal func _downCastConditional<T>(
+    into result: UnsafeMutablePointer<T>
+  ) -> Bool {
+    guard let value = _value as? T else { return false }
+    result.initialize(to: value)
+    return true
+  }
+}
+
 // ${'Local Variables'}:
 // eval: (read-only-mode 1)
 // End:

stdlib/public/core/Dictionary.swift

@@ -1515,6 +1515,61 @@ extension Dictionary: Hashable where Value: Hashable {
   }
 }
 
+extension Dictionary: _HasCustomAnyHashableRepresentation
+where Value: Hashable {
+  public func _toCustomAnyHashable() -> AnyHashable? {
+    return AnyHashable(_box: _DictionaryAnyHashableBox(self))
+  }
+}
+
+internal struct _DictionaryAnyHashableBox<Key: Hashable, Value: Hashable>
+  : _AnyHashableBox {
+  internal let _value: Dictionary<Key, Value>
+  internal let _canonical: Dictionary<AnyHashable, AnyHashable>
+
+  internal init(_ value: Dictionary<Key, Value>) {
+    self._value = value
+    self._canonical = value as Dictionary<AnyHashable, AnyHashable>
+  }
+
+  internal var _base: Any {
+    return _value
+  }
+
+  internal var _canonicalBox: _AnyHashableBox {
+    return _DictionaryAnyHashableBox<AnyHashable, AnyHashable>(_canonical)
+  }
+
+  internal func _isEqual(to other: _AnyHashableBox) -> Bool? {
+    guard
+      let other = other as? _DictionaryAnyHashableBox<AnyHashable, AnyHashable>
+    else {
+      return nil
+    }
+    return _canonical == other._value
+  }
+
+  internal var _hashValue: Int {
+    return _canonical.hashValue
+  }
+
+  internal func _hash(into hasher: inout Hasher) {
+    _canonical.hash(into: &hasher)
+  }
+
+  internal func _unbox<T: Hashable>() -> T? {
+    return _value as? T
+  }
+
+  internal func _downCastConditional<T>(
+    into result: UnsafeMutablePointer<T>
+  ) -> Bool {
+    guard let value = _value as? T else { return false }
+    result.initialize(to: value)
+    return true
+  }
+}
+
 extension Dictionary: CustomStringConvertible, CustomDebugStringConvertible {
   @inlinable // FIXME(sil-serialize-all)
   internal func _makeDescription() -> String {