Merge pull request #18096 from eeckstein/fix-benchmark-iteration-times

benchmarks: make benchmarks more stable

Author: eeckstein

benchmark/single-source/Array2D.swift

@@ -15,18 +15,21 @@ import TestsUtils
 public let Array2D = BenchmarkInfo(
   name: "Array2D",
   runFunction: run_Array2D,
-  tags: [.validation, .api, .Array])
+  tags: [.validation, .api, .Array],
+  setUpFunction: { blackHole(inputArray) },
+  tearDownFunction: { inputArray = nil })
 
-@inline(never)
-public func run_Array2D(_ N: Int) {
+var inputArray: [[Int]]! = {
   var A: [[Int]] = []
+  A.reserveCapacity(1024)
   for _ in 0 ..< 1024 {
-    var B: [Int] = []
-    for y in 0 ..< 1024 {
-      B.append(y)
-    }
-    A.append(B)
+    A.append(Array(0 ..< 1024))
   }
+  return A
+}()
+
+@inline(never)
+func modifyArray(_ A: inout [[Int]], _ N: Int) {
   for _ in 0..<N {
     for i in 0 ..< 1024 {
       for y in 0 ..< 1024 {
@@ -36,3 +39,8 @@ public func run_Array2D(_ N: Int) {
     }
   }
 }
+
+@inline(never)
+public func run_Array2D(_ N: Int) {
+  modifyArray(&inputArray, N)
+}

benchmark/single-source/ArrayOfGenericPOD.swift

@@ -40,7 +40,7 @@ class RefArray<T> {
 // elements should be a nop.
 @inline(never)
 func genEnumArray() {
-  _ = RefArray<Int?>(3)
+  blackHole(RefArray<Int?>(3))
   // should be a nop
 }
 
@@ -53,13 +53,13 @@ struct S<T> {
 }
 @inline(never)
 func genStructArray() {
-  _ = RefArray<S<Int>>(S(x:3, y:4))
+  blackHole(RefArray<S<Int>>(S(x:3, y:4)))
   // should be a nop
 }
 
 @inline(never)
 public func run_ArrayOfGenericPOD(_ N: Int) {
-  for _ in 0...N {
+  for _ in 0..<N {
     genEnumArray()
     genStructArray()
   }

benchmark/single-source/ArrayOfGenericRef.swift

@@ -47,7 +47,7 @@ class GenericRef<T> : Constructible {
 // Reference to a POD class.
 @inline(never)
 func genPODRefArray() {
-  _ = ConstructibleArray<GenericRef<Int>>(3)
+  blackHole(ConstructibleArray<GenericRef<Int>>(3))
   // should be a nop
 }
 
@@ -57,7 +57,7 @@ class Dummy {}
 @inline(never)
 func genCommonRefArray() {
   let d = Dummy()
-  _ = ConstructibleArray<GenericRef<Dummy>>(d)
+  blackHole(ConstructibleArray<GenericRef<Dummy>>(d))
   // should be a nop
 }
 
@@ -74,7 +74,7 @@ class RefArray<T> {
 @inline(never)
 func genRefEnumArray() {
   let d = Dummy()
-  _ = RefArray<Dummy?>(d)
+  blackHole(RefArray<Dummy?>(d))
   // should be a nop
 }
 
@@ -88,13 +88,13 @@ struct GenericVal<T> : Constructible {
 @inline(never)
 func genRefStructArray() {
   let d = Dummy()
-  _ = ConstructibleArray<GenericVal<Dummy>>(d)
+  blackHole(ConstructibleArray<GenericVal<Dummy>>(d))
   // should be a nop
 }
 
 @inline(never)
 public func run_ArrayOfGenericRef(_ N: Int) {
-  for _ in 0...N {
+  for _ in 0..<N {
     genPODRefArray()
     genCommonRefArray()
     genRefEnumArray()

benchmark/single-source/ArrayOfPOD.swift

@@ -33,7 +33,7 @@ class RefArray<T> {
 
 @inline(never)
 func genIntArray() {
-  _ = RefArray<Int>(3, count:200_000)
+  blackHole(RefArray<Int>(3, count:200_000))
   // should be a nop
 }
 
@@ -45,7 +45,7 @@ enum PODEnum {
 
 @inline(never)
 func genEnumArray() {
-  _ = RefArray<PODEnum>(PODEnum.Some(3))
+  blackHole(RefArray<PODEnum>(PODEnum.Some(3)))
   // should be a nop
 }
 
@@ -55,13 +55,13 @@ struct S {
 }
 @inline(never)
 func genStructArray() {
-  _ = RefArray<S>(S(x:3, y:4))
+  blackHole(RefArray<S>(S(x:3, y:4)))
   // should be a nop
 }
 
 @inline(never)
 public func run_ArrayOfPOD(_ N: Int) {
-  for _ in 0...N {
+  for _ in 0..<N {
     genIntArray()
     genEnumArray()
     genStructArray()

benchmark/single-source/ArrayOfRef.swift

@@ -48,7 +48,7 @@ class POD : Constructible {
 
 @inline(never)
 func genPODRefArray() {
-  _ = ConstructibleArray<POD>(3)
+  blackHole(ConstructibleArray<POD>(3))
   // should be a nop
 }
 
@@ -64,7 +64,7 @@ class CommonRef : Constructible {
 @inline(never)
 func genCommonRefArray() {
   let d = Dummy()
-  _ = ConstructibleArray<CommonRef>(d)
+  blackHole(ConstructibleArray<CommonRef>(d))
   // should be a nop
 }
 
@@ -85,7 +85,7 @@ class RefArray<T> {
 @inline(never)
 func genRefEnumArray() {
   let e = RefEnum.Some(Dummy())
-  _ = RefArray<RefEnum>(e)
+  blackHole(RefArray<RefEnum>(e))
   // should be a nop
 }
 
@@ -99,13 +99,13 @@ struct S : Constructible {
 @inline(never)
 func genRefStructArray() {
   let d = Dummy()
-  _ = ConstructibleArray<S>(d)
+  blackHole(ConstructibleArray<S>(d))
   // should be a nop
 }
 
 @inline(never)
 public func run_ArrayOfRef(_ N: Int) {
-  for _ in 0...N {
+  for _ in 0..<N {
     genPODRefArray()
     genCommonRefArray()
     genRefEnumArray()

benchmark/single-source/CSVParsing.swift

@@ -1,18 +1,18 @@
 import TestsUtils
 public let CSVParsing = BenchmarkInfo(
-    name: "CSVParsing",
+    name: "CSVParsing2",
     runFunction: run_CSVParsing,
     tags: [.miniapplication, .api, .String],
     setUpFunction: { buildWorkload() },
     tearDownFunction: nil)
 public let CSVParsingAlt = BenchmarkInfo(
-    name: "CSVParsingAlt",
+    name: "CSVParsingAlt2",
     runFunction: run_CSVParsingAlt,
     tags: [.miniapplication, .api, .String],
     setUpFunction: { buildWorkload() },
     tearDownFunction: nil)
 public let CSVParsingAltIndices = BenchmarkInfo(
-    name: "CSVParsingAltIndices",
+    name: "CSVParsingAltIndices2",
     runFunction: run_CSVParsingAltIndices,
     tags: [.miniapplication, .api, .String],
     setUpFunction: { buildWorkload() },
@@ -198,7 +198,7 @@ let workloadBase = """
     一,二,三,四,五,六,七,
     saquui,ta'lo,tso'i,nvgi,hisgi,sudali,galiquogi
     """
-let targetRowNumber = 200_000
+let targetRowNumber = 500
 let repeatCount = targetRowNumber / workloadBase.split(separator: "\n").count
 let workload: String = repeatElement(workloadBase, count: repeatCount).joined()
 

benchmark/single-source/CString.swift

@@ -21,15 +21,18 @@ public let CString = [
   BenchmarkInfo(name: "CStringLongAscii", runFunction: run_CStringLongAscii, tags: [.validation, .api, .String, .bridging]),
   BenchmarkInfo(name: "CStringLongNonAscii", runFunction: run_CStringLongNonAscii, tags: [.validation, .api, .String, .bridging]),
   BenchmarkInfo(name: "CStringShortAscii", runFunction: run_CStringShortAscii, tags: [.validation, .api, .String, .bridging]),
-  BenchmarkInfo(name: "StringWithCString", runFunction: run_StringWithCString, tags: [.validation, .api, .String, .bridging]),
+  BenchmarkInfo(name: "StringWithCString2", runFunction: run_StringWithCString, tags: [.validation, .api, .String, .bridging],
+      setUpFunction: { blackHole(repeatedStr) }, tearDownFunction: { repeatedStr = nil })
 ]
 
 let ascii = "Swift is a multi-paradigm, compiled programming language created for iOS, OS X, watchOS, tvOS and Linux development by Apple Inc. Swift is designed to work with Apple's Cocoa and Cocoa Touch frameworks and the large body of existing Objective-C code written for Apple products. Swift is intended to be more resilient to erroneous code (\"safer\") than Objective-C and also more concise. It is built with the LLVM compiler framework included in Xcode 6 and later and uses the Objective-C runtime, which allows C, Objective-C, C++ and Swift code to run within a single program."
 let japanese = "日本語（にほんご、にっぽんご）は、主に日本国内や日本人同士の間で使われている言語である。"
 
+var repeatedStr: String! = String(repeating: "x", count: 5 * (1 << 16))
+
 @inline(never)
 public func run_StringWithCString(_ N: Int) {
-  let str = String(repeating: "x", count: 100 * (1 << 16))
+  let str: String = repeatedStr
   for _ in 0 ..< N {
     str.withCString { blackHole($0) }
   }

benchmark/single-source/CharacterProperties.swift

@@ -22,26 +22,26 @@ import Foundation
 public let CharacterPropertiesFetch = BenchmarkInfo(
   name: "CharacterPropertiesFetch",
   runFunction: run_CharacterPropertiesFetch,
-  tags: [.validation, .api, .String])
+  tags: [.validation, .api, .String],
+  setUpFunction: { blackHole(workload) })
 
 public let CharacterPropertiesStashed = BenchmarkInfo(
   name: "CharacterPropertiesStashed",
   runFunction: run_CharacterPropertiesStashed,
   tags: [.validation, .api, .String],
-  setUpFunction: { run_CharacterPropertiesStashed(1) },
-  tearDownFunction: nil)
+  setUpFunction: { setupStash() })
 
 public let CharacterPropertiesStashedMemo = BenchmarkInfo(
   name: "CharacterPropertiesStashedMemo",
   runFunction: run_CharacterPropertiesStashedMemo,
-  tags: [.validation, .api, .String])
+  tags: [.validation, .api, .String],
+  setUpFunction: { setupMemo() })
 
 public let CharacterPropertiesPrecomputed = BenchmarkInfo(
   name: "CharacterPropertiesPrecomputed",
   runFunction: run_CharacterPropertiesPrecomputed,
   tags: [.validation, .api, .String],
-  setUpFunction: { run_CharacterPropertiesPrecomputed(1) },
-  tearDownFunction: nil)
+  setUpFunction: { setupPrecomputed() })
 
 extension Character {
   var firstScalar: UnicodeScalar { return unicodeScalars.first! }
@@ -122,6 +122,20 @@ func isCapitalizedStashed(_ c: Character) -> Bool {
   return capitalizedLetters.contains(c.firstScalar)
 }
 
+func setupStash() {
+  blackHole(workload)
+    blackHole(controlCharacters)
+    blackHole(alphanumerics)
+    blackHole(lowercaseLetters)
+    blackHole(punctuationCharacters)
+    blackHole(whitespaces)
+    blackHole(letters)
+    blackHole(uppercaseLetters)
+    blackHole(decimalDigits)
+    blackHole(newlines)
+    blackHole(capitalizedLetters)
+}
+
 // Memoize the stashed set
 var controlCharactersMemo = Set<UInt32>()
 func isControlStashedMemo(_ c: Character) -> Bool {
@@ -224,6 +238,20 @@ func isCapitalizedStashedMemo(_ c: Character) -> Bool {
   return false
 }
 
+func setupMemo() {
+  blackHole(workload)
+    blackHole(controlCharactersMemo)
+    blackHole(alphanumericsMemo)
+    blackHole(lowercaseLettersMemo)
+    blackHole(punctuationCharactersMemo)
+    blackHole(whitespacesMemo)
+    blackHole(lettersMemo)
+    blackHole(uppercaseLettersMemo)
+    blackHole(decimalDigitsMemo)
+    blackHole(newlinesMemo)
+    blackHole(capitalizedLettersMemo)
+}
+
 // Precompute whole scalar set
 var controlCharactersPrecomputed: Set<UInt32> = {
   var result = Set<UInt32>()
@@ -356,6 +384,20 @@ func isCapitalizedPrecomputed(_ c: Character) -> Bool {
   return capitalizedLettersPrecomputed.contains(c.firstScalar.value)
 }
 
+func setupPrecomputed() {
+  blackHole(workload)
+    blackHole(controlCharactersPrecomputed)
+    blackHole(alphanumericsPrecomputed)
+    blackHole(lowercaseLettersPrecomputed)
+    blackHole(punctuationCharactersPrecomputed)
+    blackHole(whitespacesPrecomputed)
+    blackHole(lettersPrecomputed)
+    blackHole(uppercaseLettersPrecomputed)
+    blackHole(decimalDigitsPrecomputed)
+    blackHole(newlinesPrecomputed)
+    blackHole(capitalizedLettersPrecomputed)
+}
+
 // Compute on the fly
 //
 // TODO: If UnicodeScalars ever exposes category, etc., implement the others!