Skip to content

Commit dbcf097

Browse files
briancroombriancroom
committed
Merge pull request #109 from briancroom/performance
[SR-1355] Add Performance Measurement APIs
2 parents c4309b9 + c4d357c commit dbcf097

File tree

10 files changed

+746
-1
lines changed

10 files changed

+746
-1
lines changed

‎Sources/XCTest/PerformanceMeter.swift

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
// This source file is part of the Swift.org open source project
2+
//
3+
// Copyright (c) 2016 Apple Inc. and the Swift project authors
4+
// Licensed under Apache License v2.0 with Runtime Library Exception
5+
//
6+
// See http://swift.org/LICENSE.txt for license information
7+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
8+
//
9+
//
10+
// PerformanceMeter.swift
11+
// Measures the performance of a block of code and reports the results.
12+
//
13+
14+
#if os(Linux) || os(FreeBSD)
15+
import Foundation
16+
#else
17+
import SwiftFoundation
18+
#endif
19+
20+
/// Describes a type that is capable of measuring some aspect of code performance
21+
/// over time.
22+
internal protocol PerformanceMetric {
23+
/// Called once per iteration immediately before the tested code is executed.
24+
/// The metric should do whatever work is required to begin a new measurement.
25+
func startMeasuring()
26+
27+
/// Called once per iteration immediately after the tested code is executed.
28+
/// The metric should do whatever work is required to finalize measurement.
29+
func stopMeasuring()
30+
31+
/// Called once, after all measurements have been taken, to provide feedback
32+
/// about the collected measurements.
33+
/// - Returns: Measurement results to present to the user.
34+
func calculateResults() -> String
35+
36+
/// Called once, after all measurements have been taken, to determine whether
37+
/// the measurements should be treated as a test failure or not.
38+
/// - Returns: A diagnostic message if the results indicate failure, else nil.
39+
func failureMessage() -> String?
40+
}
41+
42+
internal protocol PerformanceMeterDelegate {
43+
func recordMeasurements(results: String, file: StaticString, line: UInt)
44+
func recordFailure(description: String, file: StaticString, line: UInt)
45+
func recordAPIViolation(description: String, file: StaticString, line: UInt)
46+
}
47+
48+
/// - Bug: This class is intended to be `internal` but is public to work around
49+
/// a toolchain bug on Linux. See `XCTestCase._performanceMeter` for more info.
50+
public final class PerformanceMeter {
51+
enum Error: ErrorProtocol, CustomStringConvertible {
52+
case noMetrics
53+
case unknownMetric(metricName: String)
54+
case startMeasuringAlreadyCalled
55+
case stopMeasuringAlreadyCalled
56+
case startMeasuringNotCalled
57+
case stopBeforeStarting
58+
59+
var description: String {
60+
switch self {
61+
case .noMetrics: return "At least one metric must be provided to measure."
62+
case .unknownMetric(let name): return "Unknown metric: \(name)"
63+
case .startMeasuringAlreadyCalled: return "Already called startMeasuring() once this iteration."
64+
case .stopMeasuringAlreadyCalled: return "Already called stopMeasuring() once this iteration."
65+
case .startMeasuringNotCalled: return "startMeasuring() must be called during the block."
66+
case .stopBeforeStarting: return "Cannot stop measuring before starting measuring."
67+
}
68+
}
69+
}
70+
71+
internal var didFinishMeasuring: Bool {
72+
return state == .measurementFinished || state == .measurementAborted
73+
}
74+
75+
private enum State {
76+
case iterationUnstarted
77+
case iterationStarted
78+
case iterationFinished
79+
case measurementFinished
80+
case measurementAborted
81+
}
82+
private var state: State = .iterationUnstarted
83+
84+
private let metrics: [PerformanceMetric]
85+
private let delegate: PerformanceMeterDelegate
86+
private let invocationFile: StaticString
87+
private let invocationLine: UInt
88+
89+
private init(metrics: [PerformanceMetric], delegate: PerformanceMeterDelegate, file: StaticString, line: UInt) {
90+
self.metrics = metrics
91+
self.delegate = delegate
92+
self.invocationFile = file
93+
self.invocationLine = line
94+
}
95+
96+
static func measureMetrics(_ metricNames: [String], delegate: PerformanceMeterDelegate, file: StaticString = #file, line: UInt = #line, for block: @noescape (PerformanceMeter) -> Void) {
97+
do {
98+
let metrics = try self.metrics(forNames: metricNames)
99+
let meter = PerformanceMeter(metrics: metrics, delegate: delegate, file: file, line: line)
100+
meter.measure(block)
101+
} catch let e {
102+
delegate.recordAPIViolation(description: String(e), file: file, line: line)
103+
}
104+
}
105+
106+
func startMeasuring(file: StaticString = #file, line: UInt = #line) {
107+
guard state == .iterationUnstarted else {
108+
state = .measurementAborted
109+
return recordAPIViolation(.startMeasuringAlreadyCalled, file: file, line: line)
110+
}
111+
state = .iterationStarted
112+
metrics.forEach { $0.startMeasuring() }
113+
}
114+
115+
func stopMeasuring(file: StaticString = #file, line: UInt = #line) {
116+
guard state != .iterationUnstarted else {
117+
return recordAPIViolation(.stopBeforeStarting, file: file, line: line)
118+
}
119+
120+
guard state != .iterationFinished else {
121+
return recordAPIViolation(.stopMeasuringAlreadyCalled, file: file, line: line)
122+
}
123+
124+
state = .iterationFinished
125+
metrics.forEach { $0.stopMeasuring() }
126+
}
127+
128+
func abortMeasuring() {
129+
state = .measurementAborted
130+
}
131+
132+
133+
private static func metrics(forNames names: [String]) throws -> [PerformanceMetric] {
134+
guard !names.isEmpty else { throw Error.noMetrics }
135+
136+
let metricsMapping = [WallClockTimeMetric.name : WallClockTimeMetric.self]
137+
138+
return try names.map({
139+
guard let metricType = metricsMapping[$0] else { throw Error.unknownMetric(metricName: $0) }
140+
return metricType.init()
141+
})
142+
}
143+
144+
private var numberOfIterations: Int {
145+
return 10
146+
}
147+
148+
private func measure(_ block: @noescape (PerformanceMeter) -> Void) {
149+
for _ in (0..<numberOfIterations) {
150+
state = .iterationUnstarted
151+
152+
block(self)
153+
stopMeasuringIfNeeded()
154+
155+
if state == .measurementAborted { return }
156+
157+
if state == .iterationUnstarted {
158+
recordAPIViolation(.startMeasuringNotCalled, file: invocationFile, line: invocationLine)
159+
return
160+
}
161+
}
162+
state = .measurementFinished
163+
164+
recordResults()
165+
recordFailures()
166+
}
167+
168+
private func stopMeasuringIfNeeded() {
169+
if state == .iterationStarted {
170+
stopMeasuring()
171+
}
172+
}
173+
174+
private func recordResults() {
175+
for metric in metrics {
176+
delegate.recordMeasurements(results: metric.calculateResults(), file: invocationFile, line: invocationLine)
177+
}
178+
}
179+
180+
private func recordFailures() {
181+
metrics.flatMap({ $0.failureMessage() }).forEach { message in
182+
delegate.recordFailure(description: message, file: invocationFile, line: invocationLine)
183+
}
184+
}
185+
186+
private func recordAPIViolation(_ error: Error, file: StaticString, line: UInt) {
187+
state = .measurementAborted
188+
delegate.recordAPIViolation(description: String(error), file: file, line: line)
189+
}
190+
}

‎Sources/XCTest/PrintObserver.swift

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,9 @@ internal class PrintObserver: XCTestObservation {
7575
return String(round(timeInterval * 1000.0) / 1000.0)
7676
}
7777
}
78+
79+
extension PrintObserver: _XCTestObservation {
80+
func testCase(_ testCase: XCTestCase, didMeasurePerformanceResults results: String, file: StaticString, line: UInt) {
81+
printAndFlush("\(file):\(line): Test Case '\(testCase.name)' measured \(results)")
82+
}
83+
}

‎Sources/XCTest/WallClockTimeMetric.swift

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// This source file is part of the Swift.org open source project
2+
//
3+
// Copyright (c) 2016 Apple Inc. and the Swift project authors
4+
// Licensed under Apache License v2.0 with Runtime Library Exception
5+
//
6+
// See http://swift.org/LICENSE.txt for license information
7+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
8+
//
9+
//
10+
// WallClockTimeMetric.swift
11+
// Performance metric measuring how long it takes code to execute
12+
//
13+
14+
#if os(Linux) || os(FreeBSD)
15+
import Foundation
16+
#else
17+
import SwiftFoundation
18+
#endif
19+
20+
/// This metric uses the system uptime to keep track of how much time passes
21+
/// between starting and stopping measuring.
22+
internal final class WallClockTimeMetric: PerformanceMetric {
23+
static let name = "org.swift.XCTPerformanceMetric_WallClockTime"
24+
25+
typealias Measurement = NSTimeInterval
26+
private var startTime: NSTimeInterval?
27+
var measurements: [Measurement] = []
28+
29+
func startMeasuring() {
30+
startTime = currentTime()
31+
}
32+
33+
func stopMeasuring() {
34+
guard let startTime = startTime else { fatalError("Must start measuring before stopping measuring") }
35+
let stopTime = currentTime()
36+
measurements.append(stopTime-startTime)
37+
}
38+
39+
private let maxRelativeStandardDeviation = 10.0
40+
private let standardDeviationNegligibilityThreshold = 0.1
41+
42+
func calculateResults() -> String {
43+
let results = [
44+
String(format: "average: %.3f", measurements.average),
45+
String(format: "relative standard deviation: %.3f%%", measurements.relativeStandardDeviation),
46+
"values: [\(measurements.map({ String(format: "%.6f", $0) }).joined(separator: ", "))]",
47+
"performanceMetricID:\(self.dynamicType.name)",
48+
String(format: "maxPercentRelativeStandardDeviation: %.3f%%", maxRelativeStandardDeviation),
49+
String(format: "maxStandardDeviation: %.3f", standardDeviationNegligibilityThreshold),
50+
]
51+
return "[Time, seconds] \(results.joined(separator: ", "))"
52+
}
53+
54+
func failureMessage() -> String? {
55+
// The relative standard deviation of the measurements is \d+.\d{3}% which is higher than the max allowed of \d+.\d{3}%.
56+
let relativeStandardDeviation = measurements.relativeStandardDeviation
57+
if (relativeStandardDeviation > maxRelativeStandardDeviation &&
58+
measurements.standardDeviation > standardDeviationNegligibilityThreshold) {
59+
return String(format: "The relative standard deviation of the measurements is %.3f%% which is higher than the max allowed of %.3f%%.", relativeStandardDeviation, maxRelativeStandardDeviation)
60+
}
61+
62+
return nil
63+
}
64+
65+
private func currentTime() -> NSTimeInterval {
66+
return NSProcessInfo.processInfo().systemUptime
67+
}
68+
}
69+
70+
71+
private extension Collection where Index: IntegerLiteralConvertible, Iterator.Element == WallClockTimeMetric.Measurement {
72+
var average: WallClockTimeMetric.Measurement {
73+
return self.reduce(0, combine: +) / Double(count.toIntMax())
74+
}
75+
76+
var standardDeviation: WallClockTimeMetric.Measurement {
77+
let average = self.average
78+
let squaredDifferences = self.map({ pow($0 - average, 2.0) })
79+
let variance = squaredDifferences.reduce(0, combine: +) / Double(count.toIntMax()-1)
80+
return sqrt(variance)
81+
}
82+
83+
var relativeStandardDeviation: Double {
84+
return (standardDeviation*100) / average
85+
}
86+
}

0 commit comments

Comments
 (0)