[SR-1355] Add Performance Measurement APIs

Sources/XCTest/PerformanceMeter.swift

@@ -0,0 +1,190 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  PerformanceMeter.swift
+//  Measures the performance of a block of code and reports the results.
+//
+
+#if os(Linux) || os(FreeBSD)
+    import Foundation
+#else
+    import SwiftFoundation
+#endif
+
+/// Describes a type that is capable of measuring some aspect of code performance
+/// over time.
+internal protocol PerformanceMetric {
+    /// Called once per iteration immediately before the tested code is executed. 
+    /// The metric should do whatever work is required to begin a new measurement.
+    func startMeasuring()
+
+    /// Called once per iteration immediately after the tested code is executed.
+    /// The metric should do whatever work is required to finalize measurement.
+    func stopMeasuring()
+
+    /// Called once, after all measurements have been taken, to provide feedback
+    /// about the collected measurements.
+    /// - Returns: Measurement results to present to the user.
+    func calculateResults() -> String
+
+    /// Called once, after all measurements have been taken, to determine whether
+    /// the measurements should be treated as a test failure or not.
+    /// - Returns: A diagnostic message if the results indicate failure, else nil.
+    func failureMessage() -> String?
+}
+
+internal protocol PerformanceMeterDelegate {
+    func recordMeasurements(results: String, file: StaticString, line: UInt)
+    func recordFailure(description: String, file: StaticString, line: UInt)
+    func recordAPIViolation(description: String, file: StaticString, line: UInt)
+}
+
+/// - Bug: This class is intended to be `internal` but is public to work around
+/// a toolchain bug on Linux. See `XCTestCase._performanceMeter` for more info.
+public final class PerformanceMeter {
+    enum Error: ErrorProtocol, CustomStringConvertible {
+        case noMetrics
+        case unknownMetric(metricName: String)
+        case startMeasuringAlreadyCalled
+        case stopMeasuringAlreadyCalled
+        case startMeasuringNotCalled
+        case stopBeforeStarting
+
+        var description: String {
+            switch self {
+            case .noMetrics: return "At least one metric must be provided to measure."
+            case .unknownMetric(let name): return "Unknown metric: \(name)"
+            case .startMeasuringAlreadyCalled: return "Already called startMeasuring() once this iteration."
+            case .stopMeasuringAlreadyCalled: return "Already called stopMeasuring() once this iteration."
+            case .startMeasuringNotCalled: return "startMeasuring() must be called during the block."
+            case .stopBeforeStarting: return "Cannot stop measuring before starting measuring."
+            }
+        }
+    }
+
+    internal var didFinishMeasuring: Bool {
+        return state == .measurementFinished || state == .measurementAborted
+    }
+
+    private enum State {
+        case iterationUnstarted
+        case iterationStarted
+        case iterationFinished
+        case measurementFinished
+        case measurementAborted
+    }
+    private var state: State = .iterationUnstarted
+
+    private let metrics: [PerformanceMetric]
+    private let delegate: PerformanceMeterDelegate
+    private let invocationFile: StaticString
+    private let invocationLine: UInt
+
+    private init(metrics: [PerformanceMetric], delegate: PerformanceMeterDelegate, file: StaticString, line: UInt) {
+        self.metrics = metrics
+        self.delegate = delegate
+        self.invocationFile = file
+        self.invocationLine = line
+    }
+
+    static func measureMetrics(_ metricNames: [String], delegate: PerformanceMeterDelegate, file: StaticString = #file, line: UInt = #line, for block: @noescape (PerformanceMeter) -> Void) {
+        do {
+            let metrics = try self.metrics(forNames: metricNames)
+            let meter = PerformanceMeter(metrics: metrics, delegate: delegate, file: file, line: line)
+            meter.measure(block)
+        } catch let e {
+            delegate.recordAPIViolation(description: String(e), file: file, line: line)
+        }
+    }
+
+    func startMeasuring(file: StaticString = #file, line: UInt = #line) {
+        guard state == .iterationUnstarted else {
+            state = .measurementAborted
+            return recordAPIViolation(.startMeasuringAlreadyCalled, file: file, line: line)
+        }
+        state = .iterationStarted
+        metrics.forEach { $0.startMeasuring() }
+    }
+
+    func stopMeasuring(file: StaticString = #file, line: UInt = #line) {
+        guard state != .iterationUnstarted else {
+            return recordAPIViolation(.stopBeforeStarting, file: file, line: line)
+        }
+
+        guard state != .iterationFinished else {
+            return recordAPIViolation(.stopMeasuringAlreadyCalled, file: file, line: line)
+        }
+
+        state = .iterationFinished
+        metrics.forEach { $0.stopMeasuring() }
+    }
+
+    func abortMeasuring() {
+        state = .measurementAborted
+    }
+
+
+    private static func metrics(forNames names: [String]) throws -> [PerformanceMetric] {
+        guard !names.isEmpty else { throw Error.noMetrics }
+
+        let metricsMapping = [WallClockTimeMetric.name : WallClockTimeMetric.self]
+
+        return try names.map({
+            guard let metricType = metricsMapping[$0] else { throw Error.unknownMetric(metricName: $0) }
+            return metricType.init()
+        })
+    }
+
+    private var numberOfIterations: Int {
+        return 10
+    }
+
+    private func measure(_ block: @noescape (PerformanceMeter) -> Void) {
+        for _ in (0..<numberOfIterations) {
+            state = .iterationUnstarted
+
+            block(self)
+            stopMeasuringIfNeeded()
+
+            if state == .measurementAborted { return }
+
+            if state == .iterationUnstarted {
+                recordAPIViolation(.startMeasuringNotCalled, file: invocationFile, line: invocationLine)
+                return
+            }
+        }
+        state = .measurementFinished
+
+        recordResults()
+        recordFailures()
+    }
+
+    private func stopMeasuringIfNeeded() {
+        if state == .iterationStarted {
+            stopMeasuring()
+        }
+    }
+
+    private func recordResults() {
+        for metric in metrics {
+            delegate.recordMeasurements(results: metric.calculateResults(), file: invocationFile, line: invocationLine)
+        }
+    }
+
+    private func recordFailures() {
+        metrics.flatMap({ $0.failureMessage() }).forEach { message in
+            delegate.recordFailure(description: message, file: invocationFile, line: invocationLine)
+        }
+    }
+
+    private func recordAPIViolation(_ error: Error, file: StaticString, line: UInt) {
+        state = .measurementAborted
+        delegate.recordAPIViolation(description: String(error), file: file, line: line)
+    }
+}

Sources/XCTest/PrintObserver.swift

@@ -75,3 +75,9 @@ internal class PrintObserver: XCTestObservation {
         return String(round(timeInterval * 1000.0) / 1000.0)
     }
 }
+
+extension PrintObserver: _XCTestObservation {
+    func testCase(_ testCase: XCTestCase, didMeasurePerformanceResults results: String, file: StaticString, line: UInt) {
+        printAndFlush("\(file):\(line): Test Case '\(testCase.name)' measured \(results)")
+    }
+}

Sources/XCTest/WallClockTimeMetric.swift

@@ -0,0 +1,86 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//
+//  WallClockTimeMetric.swift
+//  Performance metric measuring how long it takes code to execute
+//
+
+#if os(Linux) || os(FreeBSD)
+    import Foundation
+#else
+    import SwiftFoundation
+#endif
+
+/// This metric uses the system uptime to keep track of how much time passes
+/// between starting and stopping measuring.
+internal final class WallClockTimeMetric: PerformanceMetric {
+    static let name = "org.swift.XCTPerformanceMetric_WallClockTime"
+
+    typealias Measurement = NSTimeInterval
+    private var startTime: NSTimeInterval?
+    var measurements: [Measurement] = []
+
+    func startMeasuring() {
+        startTime = currentTime()
+    }
+
+    func stopMeasuring() {
+        guard let startTime = startTime else { fatalError("Must start measuring before stopping measuring") }
+        let stopTime = currentTime()
+        measurements.append(stopTime-startTime)
+    }
+
+    private let maxRelativeStandardDeviation = 10.0
+    private let standardDeviationNegligibilityThreshold = 0.1
+
+    func calculateResults() -> String {
+        let results = [
+                          String(format: "average: %.3f", measurements.average),
+                          String(format: "relative standard deviation: %.3f%%", measurements.relativeStandardDeviation),
+                          "values: [\(measurements.map({ String(format: "%.6f", $0) }).joined(separator: ", "))]",
+                          "performanceMetricID:\(self.dynamicType.name)",
+                          String(format: "maxPercentRelativeStandardDeviation: %.3f%%", maxRelativeStandardDeviation),
+                          String(format: "maxStandardDeviation: %.3f", standardDeviationNegligibilityThreshold),
+                          ]
+        return "[Time, seconds] \(results.joined(separator: ", "))"
+    }
+
+    func failureMessage() -> String? {
+        // The relative standard deviation of the measurements is \d+.\d{3}% which is higher than the max allowed of \d+.\d{3}%.
+        let relativeStandardDeviation = measurements.relativeStandardDeviation
+        if (relativeStandardDeviation > maxRelativeStandardDeviation &&
+            measurements.standardDeviation > standardDeviationNegligibilityThreshold) {
+            return String(format: "The relative standard deviation of the measurements is %.3f%% which is higher than the max allowed of %.3f%%.", relativeStandardDeviation, maxRelativeStandardDeviation)
+        }
+
+        return nil
+    }
+
+    private func currentTime() -> NSTimeInterval {
+        return NSProcessInfo.processInfo().systemUptime
+    }
+}
+
+
+private extension Collection where Index: IntegerLiteralConvertible, Iterator.Element == WallClockTimeMetric.Measurement {
+    var average: WallClockTimeMetric.Measurement {
+        return self.reduce(0, combine: +) / Double(count.toIntMax())
+    }
+
+    var standardDeviation: WallClockTimeMetric.Measurement {
+        let average = self.average
+        let squaredDifferences = self.map({ pow($0 - average, 2.0) })
+        let variance = squaredDifferences.reduce(0, combine: +) / Double(count.toIntMax()-1)
+        return sqrt(variance)
+    }
+
+    var relativeStandardDeviation: Double {
+        return (standardDeviation*100) / average
+    }
+}