Merge pull request #15207 from lorentey/seedless-hashes

[runtime] Add better control over the random hashing seed

Author: lorentey

stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb

@@ -1091,14 +1091,6 @@ struct PersistentState {
   static var complaintInstalled = false
   static var hashingKeyOverridden = false
 
-  static func overrideHashingKey() {
-    if !hashingKeyOverridden {
-      // FIXME(hasher): This has to run before creating the first Set/Dictionary
-      _Hasher._secretKey = (0, 0)
-      hashingKeyOverridden = true
-    }
-  }
-
   static func complainIfNothingRuns() {
     if !complaintInstalled {
       complaintInstalled = true
@@ -1209,7 +1201,6 @@ func stopTrackingObjects(_: UnsafePointer<CChar>) -> Int
 
 public final class TestSuite {
   public init(_ name: String) {
-    PersistentState.overrideHashingKey()
     self.name = name
     _precondition(
       _testNameToIndex[name] == nil,

stdlib/public/SwiftShims/GlobalObjects.h

@@ -19,6 +19,7 @@
 #define SWIFT_STDLIB_SHIMS_GLOBALOBJECTS_H_
 
 #include "SwiftStdint.h"
+#include "SwiftStdbool.h"
 #include "HeapObject.h"
 #include "Visibility.h"
 
@@ -77,16 +78,16 @@ struct _SwiftEmptyDictionaryStorage _swiftEmptyDictionaryStorage;
 SWIFT_RUNTIME_STDLIB_INTERFACE
 struct _SwiftEmptySetStorage _swiftEmptySetStorage;
 
-struct _SwiftHashingSecretKey {
-  __swift_uint64_t key0;
-  __swift_uint64_t key1;
+struct _SwiftHashingSeed {
+  __swift_uint64_t seed0;
+  __swift_uint64_t seed1;
 };
 
 SWIFT_RUNTIME_STDLIB_INTERFACE
-struct _SwiftHashingSecretKey _swift_stdlib_Hashing_secretKey;
+struct _SwiftHashingSeed _swift_stdlib_Hashing_seed;
 
 SWIFT_RUNTIME_STDLIB_INTERFACE
-__swift_uint64_t _swift_stdlib_HashingDetail_fixedSeedOverride;
+__swift_bool _swift_stdlib_Hashing_deterministicHashing;
 
 #ifdef __cplusplus
 

stdlib/public/core/Hashing.swift

@@ -27,30 +27,14 @@ import SwiftShims
 public // @testable
 enum _HashingDetail {
 
-  // FIXME(hasher): Remove
-  @_inlineable // FIXME(sil-serialize-all)
-  public // @testable
-  static var fixedSeedOverride: UInt64 {
-    get {
-      // HACK: the variable itself is defined in C++ code so that it is
-      // guaranteed to be statically initialized.  This is a temporary
-      // workaround until the compiler can do the same for Swift.
-      return _swift_stdlib_HashingDetail_fixedSeedOverride
-    }
-    set {
-      _swift_stdlib_HashingDetail_fixedSeedOverride = newValue
-    }
-  }
-
   // FIXME(hasher): Remove
   @_inlineable // FIXME(sil-serialize-all)
   @_versioned
   @_transparent
   internal static func getExecutionSeed() -> UInt64 {
     // FIXME: This needs to be a per-execution seed. This is just a placeholder
     // implementation.
-    let seed: UInt64 = 0xff51afd7ed558ccd
-    return _HashingDetail.fixedSeedOverride == 0 ? seed : fixedSeedOverride
+    return 0xff51afd7ed558ccd
   }
 
   // FIXME(hasher): Remove
@@ -224,33 +208,39 @@ public struct _Hasher {
   // NOT @_inlineable
   @effects(releasenone)
   public init() {
-    self._core = Core(key: _Hasher._secretKey)
+    self._core = Core(key: _Hasher._seed)
   }
 
   // NOT @_inlineable
   @effects(releasenone)
-  public init(key: (UInt64, UInt64)) {
-    self._core = Core(key: key)
+  public init(seed: (UInt64, UInt64)) {
+    self._core = Core(key: seed)
   }
 
-  // FIXME(ABI)#41 : make this an actual public API.
-  @_inlineable // FIXME(sil-serialize-all)
+  /// Indicates whether we're running in an environment where hashing needs to
+  /// be deterministic. If this is true, the hash seed is not random, and hash
+  /// tables do not apply per-instance perturbation that is not repeatable.
+  /// This is not recommended for production use, but it is useful in certain
+  /// test environments where randomization may lead to unwanted nondeterminism
+  /// of test results.
+  public // SPI
+  static var _isDeterministic: Bool {
+    return _swift_stdlib_Hashing_deterministicHashing
+  }
+
+  /// The 128-bit hash seed used to initialize the hasher state. Initialized
+  /// once during process startup.
   public // SPI
-  static var _secretKey: (UInt64, UInt64) {
+  static var _seed: (UInt64, UInt64) {
     get {
-      // The variable itself is defined in C++ code so that it is initialized
-      // during static construction.  Almost every Swift program uses hash
-      // tables, so initializing the secret key during the startup seems to be
-      // the right trade-off.
+      if _isDeterministic { return (0, 0) }
+      // The seed itself is defined in C++ code so that it is initialized during
+      // static construction.  Almost every Swift program uses hash tables, so
+      // initializing the seed during the startup seems to be the right
+      // trade-off.
       return (
-        _swift_stdlib_Hashing_secretKey.key0,
-        _swift_stdlib_Hashing_secretKey.key1)
-    }
-    set {
-      // FIXME(hasher) Replace setter with some override mechanism inside 
-      // the runtime
-      (_swift_stdlib_Hashing_secretKey.key0,
-       _swift_stdlib_Hashing_secretKey.key1) = newValue
+        _swift_stdlib_Hashing_seed.seed0,
+        _swift_stdlib_Hashing_seed.seed1)
     }
   }
 

stdlib/public/stubs/GlobalObjects.cpp

@@ -106,20 +106,37 @@ swift::_SwiftEmptySetStorage swift::_swiftEmptySetStorage = {
   0 // int entries; (zero'd bits)
 };
 
-static __swift_uint64_t randomUInt64() {
+static swift::_SwiftHashingSeed initializeHashingSeed() {
+#if defined(__APPLE__)
+  // Use arc4random if available.
+  swift::_SwiftHashingSeed seed = { 0, 0 };
+  arc4random_buf(&seed, sizeof(seed));
+  return seed;
+#else
   std::random_device randomDevice;
-  std::mt19937_64 twisterEngine(randomDevice());
+  std::mt19937_64 engine(randomDevice());
   std::uniform_int_distribution<__swift_uint64_t> distribution;
-  return distribution(twisterEngine);
+  return { distribution(engine), distribution(engine) };
+#endif
+}
+
+static __swift_bool initializeHashingDeterminism() {
+  // Setting the environment variable SWIFT_DETERMINISTIC_HASHING to "1"
+  // disables randomized hash seeding. This is useful in cases we need to ensure
+  // results are repeatable, e.g., in certain test environments.  (Note that
+  // even if the seed override is enabled, hash values aren't guaranteed to
+  // remain stable across even minor stdlib releases.)
+  auto determinism = getenv("SWIFT_DETERMINISTIC_HASHING");
+  return determinism && 0 == strcmp(determinism, "1");
 }
 
 SWIFT_ALLOWED_RUNTIME_GLOBAL_CTOR_BEGIN
-swift::_SwiftHashingSecretKey swift::_swift_stdlib_Hashing_secretKey = {
-  randomUInt64(), randomUInt64()
-};
+swift::_SwiftHashingSeed swift::_swift_stdlib_Hashing_seed =
+  initializeHashingSeed();
+__swift_bool swift::_swift_stdlib_Hashing_deterministicHashing =
+  initializeHashingDeterminism();
 SWIFT_ALLOWED_RUNTIME_GLOBAL_CTOR_END
 
-__swift_uint64_t swift::_swift_stdlib_HashingDetail_fixedSeedOverride = 0;
 
 SWIFT_RUNTIME_STDLIB_INTERFACE
 void swift::_swift_instantiateInertHeapObject(void *address,

test/lit.cfg

@@ -914,7 +914,8 @@ ENV_VAR_PREFIXES = {
     'watchsimulator': SIMULATOR_ENV_PREFIX,
     'appletvsimulator': SIMULATOR_ENV_PREFIX
 }
-config.substitutions.append(('%env-', ENV_VAR_PREFIXES.get(config.target_sdk_name, "")))
+TARGET_ENV_PREFIX = ENV_VAR_PREFIXES.get(config.target_sdk_name, "")
+config.substitutions.append(('%env-', TARGET_ENV_PREFIX))
 config.substitutions.append(("%target-sdk-name", config.target_sdk_name))
 
 config.compiler_rt_libs = []
@@ -1134,6 +1135,10 @@ if config.lldb_build_root != "":
     python_lib_dir = get_python_lib(True, False, config.lldb_build_root)
     config.substitutions.append(('%lldb-python-path', python_lib_dir))
 
+# Disable randomized hash seeding by default. Tests need to manually opt in to
+# random seeds by unsetting the SWIFT_DETERMINISTIC_HASHING environment
+# variable.
+config.environment[TARGET_ENV_PREFIX + 'SWIFT_DETERMINISTIC_HASHING'] = '1'
 
 # Run lsb_release on the target to be tested and return the results.
 def linux_get_lsb_release():

validation-test/stdlib/FixedPoint.swift.gyb

@@ -242,7 +242,7 @@ FixedPoint.test("${Self}.hashValue") {
     let input = get${Self}(${input})
     let output = getInt(input.hashValue)
 
-    var hasher = _SipHash13(key: _Hasher._secretKey)
+    var hasher = _SipHash13(key: _Hasher._seed)
 %     if prepare_bit_pattern(input, word_bits, self_ty.is_signed) == input:
     hasher.append(${input} as ${"" if self_ty.is_signed else "U"}Int)
 %     else:

validation-test/stdlib/Hashing.swift

@@ -10,11 +10,11 @@ var HashingTestSuite = TestSuite("Hashing")
 
 func checkHash(
   for value: UInt64,
-  withKey key: (UInt64, UInt64),
+  withSeed seed: (UInt64, UInt64),
   expected: UInt64,
   file: String = #file, line: UInt = #line
 ) {
-  var hasher = _Hasher(key: key)
+  var hasher = _Hasher(seed: seed)
   hasher.append(bits: value)
   let hash = hasher.finalize()
   expectEqual(
@@ -24,23 +24,23 @@ func checkHash(
 
 HashingTestSuite.test("_Hasher/CustomKeys") {
   // This assumes _Hasher implements SipHash-1-3.
-  checkHash(for: 0, withKey: (0, 0), expected: 0xbd60acb658c79e45)
-  checkHash(for: 0, withKey: (0, 1), expected: 0x1ce32b0b44e61175)
-  checkHash(for: 0, withKey: (1, 0), expected: 0x9c44b7c8df2ca74b)
-  checkHash(for: 0, withKey: (1, 1), expected: 0x9653ca0a3b455506)
-  checkHash(for: 0, withKey: (.max, .max), expected: 0x3ab336a4895e4d36)
-
-  checkHash(for: 1, withKey: (0, 0), expected: 0x1e9f734161d62dd9)
-  checkHash(for: 1, withKey: (0, 1), expected: 0xb6fcf32d09f76cba)
-  checkHash(for: 1, withKey: (1, 0), expected: 0xacb556b13007504a)
-  checkHash(for: 1, withKey: (1, 1), expected: 0x7defec680db51d24)
-  checkHash(for: 1, withKey: (.max, .max), expected: 0x212798441870ef6b)
-
-  checkHash(for: .max, withKey: (0, 0), expected: 0x2f205be2fec8e38d)
-  checkHash(for: .max, withKey: (0, 1), expected: 0x3ff7fa33381ecf7b)
-  checkHash(for: .max, withKey: (1, 0), expected: 0x404afd8eb2c4b22a)
-  checkHash(for: .max, withKey: (1, 1), expected: 0x855642d657c1bd46)
-  checkHash(for: .max, withKey: (.max, .max), expected: 0x5b16b7a8181980c2)
+  checkHash(for: 0, withSeed: (0, 0), expected: 0xbd60acb658c79e45)
+  checkHash(for: 0, withSeed: (0, 1), expected: 0x1ce32b0b44e61175)
+  checkHash(for: 0, withSeed: (1, 0), expected: 0x9c44b7c8df2ca74b)
+  checkHash(for: 0, withSeed: (1, 1), expected: 0x9653ca0a3b455506)
+  checkHash(for: 0, withSeed: (.max, .max), expected: 0x3ab336a4895e4d36)
+
+  checkHash(for: 1, withSeed: (0, 0), expected: 0x1e9f734161d62dd9)
+  checkHash(for: 1, withSeed: (0, 1), expected: 0xb6fcf32d09f76cba)
+  checkHash(for: 1, withSeed: (1, 0), expected: 0xacb556b13007504a)
+  checkHash(for: 1, withSeed: (1, 1), expected: 0x7defec680db51d24)
+  checkHash(for: 1, withSeed: (.max, .max), expected: 0x212798441870ef6b)
+
+  checkHash(for: .max, withSeed: (0, 0), expected: 0x2f205be2fec8e38d)
+  checkHash(for: .max, withSeed: (0, 1), expected: 0x3ff7fa33381ecf7b)
+  checkHash(for: .max, withSeed: (1, 0), expected: 0x404afd8eb2c4b22a)
+  checkHash(for: .max, withSeed: (1, 1), expected: 0x855642d657c1bd46)
+  checkHash(for: .max, withSeed: (.max, .max), expected: 0x5b16b7a8181980c2)
 }
 
 HashingTestSuite.test("_Hasher/DefaultKey") {
@@ -52,22 +52,15 @@ HashingTestSuite.test("_Hasher/DefaultKey") {
   defaultHasher.append(bits: value)
   expectEqual(defaultHasher.finalize(), defaultHash)
 
-  var customHasher = _Hasher(key: _Hasher._secretKey)
+  var customHasher = _Hasher(seed: _Hasher._seed)
   customHasher.append(bits: value)
   expectEqual(customHasher.finalize(), defaultHash)
 }
 
-HashingTestSuite.test("_Hasher/keyOverride") {
-  let value: UInt64 = 0x0102030405060708
-  let expected = Int(truncatingIfNeeded: 0x661dac5d71c78013 as UInt64)
-
-  let originalKey = _Hasher._secretKey
-  _Hasher._secretKey = (1, 2)
-  let hash = _hashValue(for: value)
-  _Hasher._secretKey = originalKey
-
-  expectEqual(hash, expected)
+HashingTestSuite.test("_Hasher/determinism") {
+  // By defaults, tests are configured to run with deterministic hashing.
+  expectTrue(_Hasher._isDeterministic)
+  expectEqual((0, 0), _Hasher._seed)
 }
 
 runAllTests()
-

validation-test/stdlib/HashingRandomization.swift

@@ -0,0 +1,26 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift -module-name main %s -o %t/hash
+// RUN: (export -n %env-SWIFT_DETERMINISTIC_HASHING; %target-run %t/hash && %target-run %t/hash) | %FileCheck %s
+// REQUIRES: executable_test
+
+// This check verifies that the hash seed is randomly generated on every
+// execution of a Swift program. There is a minuscule chance that the same seed
+// is generated on two separate executions; however, a test failure here is more
+// likely to indicate an issue with the random number generator or the testing
+// environment.
+
+print("Deterministic: \(_Hasher._isDeterministic)")
+print("Seed: \(_Hasher._seed)")
+print("Hash values: <\(0.hashValue), \(1.hashValue)>")
+
+// On the first run, remember the seed and hash value.
+//   CHECK: Deterministic: false
+//   CHECK-NEXT: Seed: [[SEED0:\([0-9]+, [0-9]+\)]]
+//   CHECK-NEXT: Hash values: [[HASH0:<-?[0-9]+, -?[0-9]+>]]
+
+// Check that the values are different on the second run.
+//   CHECK-NEXT: Deterministic: false
+//   CHECK-NEXT: Seed:
+//   CHECK-NOT: [[SEED0]]
+//   CHECK-NEXT: Hash values:
+//   CHECK-NOT: [[HASH0]]