Skip to content

Commit e47c7db

Browse files
benlangmuirbenlangmuir
committed
[test] Disable performance metrics unless ENABLE_PERF_TESTS is defined
When running in debug, or in continuous integration that is not setup for performance testing, default to not collecting performance metrics to avoid failures due to slow or high-variability tests (default max_stddev = 10% in xctest). Instead, run the measured block once and skip the timing code. This has the nice side effect of speeding up test runs in debug builds.
1 parent e8909fd commit e47c7db

File tree

6 files changed

+156
-76
lines changed

6 files changed

+156
-76
lines changed

‎Package.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,6 @@ let package = Package(
8282
dependencies: ["SwiftPM"]),
8383
.testTarget(
8484
name: "SKSupportTests",
85-
dependencies: ["SKSupport"]),
85+
dependencies: ["SKSupport", "SKTestSupport"]),
8686
]
8787
)

‎Sources/SKTestSupport/PerfTestCase.swift

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import XCTest
14+
15+
/// Base class for a performance test case in SourceKit-LSP.
16+
///
17+
/// This allows writing performance tests whose performance tracking is only
18+
/// enabled when ENABLE_PERF_TESTS is defined. Otherwise, the test is still
19+
/// executed, but no metrics are enabled, and the measured block is only run
20+
/// once, which is useful to avoid failures due to high variability in
21+
/// continuous integration.
22+
open class PerfTestCase: XCTestCase {
23+
24+
#if !ENABLE_PERF_TESTS
25+
26+
#if os(macOS)
27+
open override func startMeasuring() {}
28+
open override func stopMeasuring() {}
29+
open override func measureMetrics(
30+
_: [XCTPerformanceMetric],
31+
automaticallyStartMeasuring: Bool,
32+
for block: () -> Void)
33+
{
34+
block()
35+
}
36+
#else
37+
// In corelibs-xctest, these methods are public, not open, so we can only
38+
// shadow them.
39+
public func startMeasuring() {}
40+
public func stopMeasuring() {}
41+
public func measureMetrics(
42+
_: [XCTPerformanceMetric],
43+
automaticallyStartMeasuring: Bool,
44+
for block: () -> Void)
45+
{
46+
block()
47+
}
48+
#endif
49+
#endif
50+
51+
}

‎Tests/LanguageServerProtocolJSONRPCTests/ConnectionPerfTests.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import LanguageServerProtocolJSONRPC
1515
import SKTestSupport
1616
import XCTest
1717

18-
class ConnectionPerfTests: XCTestCase {
18+
class ConnectionPerfTests: PerfTestCase {
1919

2020
var connection: TestJSONRPCConnection! = nil
2121

‎Tests/SKSupportTests/SupportPerfTests.swift

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
@testable import SKSupport
14+
import SKTestSupport
15+
import Basic
16+
import POSIX
17+
import XCTest
18+
19+
final class SupportPerfTests: PerfTestCase {
20+
21+
func testLineTableAppendPerf() {
22+
23+
let characters: [Character] = [
24+
"\t", "\n"
25+
] + (32...126).map { Character(UnicodeScalar($0)) }
26+
27+
// The debug performance is shockingly bad.
28+
#if DEBUG
29+
let iterations = 1_000
30+
#else
31+
let iterations = 10_000
32+
#endif
33+
34+
self.measure {
35+
var lcg = SimpleLCG(seed: 1)
36+
var t = LineTable("")
37+
var line = 0
38+
var col = 0
39+
for _ in 1...iterations {
40+
let c = characters.randomElement(using: &lcg)!
41+
t.replace(fromLine: line, utf16Offset: col, toLine: line, utf16Offset: col, with: String(c))
42+
col += 1
43+
if c == "\n" {
44+
line += 1
45+
col = 0
46+
}
47+
}
48+
}
49+
}
50+
51+
func testLineTableSingleCharEditPerf() {
52+
53+
let characters: [Character] = [
54+
"\t", "\n"
55+
] + (32...126).map { Character(UnicodeScalar($0)) }
56+
57+
// The debug performance is shockingly bad.
58+
#if DEBUG
59+
let iterations = 1_000
60+
let size = 1_000
61+
#else
62+
let iterations = 10_000
63+
let size = 10_000
64+
#endif
65+
66+
self.measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
67+
var lcg = SimpleLCG(seed: 1)
68+
var str = ""
69+
for _ in 1...size {
70+
str += String(characters.randomElement(using: &lcg)!)
71+
}
72+
73+
var t = LineTable(str)
74+
75+
self.startMeasuring()
76+
77+
for _ in 1...iterations {
78+
let line = (0..<(t.count-1)).randomElement(using: &lcg)!
79+
let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg)!
80+
let len = Bool.random() ? 1 : 0
81+
var newText = String(characters.randomElement(using: &lcg)!)
82+
if len == 1 && Bool.random(using: &lcg) {
83+
newText = "" // deletion
84+
}
85+
86+
t.replace(fromLine: line, utf16Offset: col, utf16Length: len, with: newText)
87+
}
88+
89+
self.stopMeasuring()
90+
}
91+
}
92+
}

‎Tests/SKSupportTests/SupportTests.swift

Lines changed: 0 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -439,78 +439,6 @@ final class SupportTests: XCTestCase {
439439
XCTAssertEqual(t, LineTable("abp\nq\ngh"))
440440
}
441441

442-
func testLineTableAppendPerf() {
443-
444-
let characters: [Character] = [
445-
"\t", "\n"
446-
] + (32...126).map { Character(UnicodeScalar($0)) }
447-
448-
// The debug performance is shockingly bad.
449-
#if DEBUG
450-
let iterations = 1_000
451-
#else
452-
let iterations = 10_000
453-
#endif
454-
455-
self.measure {
456-
var lcg = SimpleLCG(seed: 1)
457-
var t = LineTable("")
458-
var line = 0
459-
var col = 0
460-
for _ in 1...iterations {
461-
let c = characters.randomElement(using: &lcg)!
462-
t.replace(fromLine: line, utf16Offset: col, toLine: line, utf16Offset: col, with: String(c))
463-
col += 1
464-
if c == "\n" {
465-
line += 1
466-
col = 0
467-
}
468-
}
469-
}
470-
}
471-
472-
func testLineTableSingleCharEditPerf() {
473-
474-
let characters: [Character] = [
475-
"\t", "\n"
476-
] + (32...126).map { Character(UnicodeScalar($0)) }
477-
478-
// The debug performance is shockingly bad.
479-
#if DEBUG
480-
let iterations = 1_000
481-
let size = 1_000
482-
#else
483-
let iterations = 10_000
484-
let size = 10_000
485-
#endif
486-
487-
self.measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
488-
var lcg = SimpleLCG(seed: 1)
489-
var str = ""
490-
for _ in 1...size {
491-
str += String(characters.randomElement(using: &lcg)!)
492-
}
493-
494-
var t = LineTable(str)
495-
496-
self.startMeasuring()
497-
498-
for _ in 1...iterations {
499-
let line = (0..<(t.count-1)).randomElement(using: &lcg)!
500-
let col = (0 ..< t[line].utf16.count).randomElement(using: &lcg)!
501-
let len = Bool.random() ? 1 : 0
502-
var newText = String(characters.randomElement(using: &lcg)!)
503-
if len == 1 && Bool.random(using: &lcg) {
504-
newText = "" // deletion
505-
}
506-
507-
t.replace(fromLine: line, utf16Offset: col, utf16Length: len, with: newText)
508-
}
509-
510-
self.stopMeasuring()
511-
}
512-
}
513-
514442
func testByteStringWithUnsafeData() {
515443
ByteString(encodingAsUTF8: "").withUnsafeData { data in
516444
XCTAssertEqual(data.count, 0)

‎Tests/SKSupportTests/XCTestManifests.swift

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
#if !canImport(ObjectiveC)
22
import XCTest
33

4+
extension SupportPerfTests {
5+
// DO NOT MODIFY: This is autogenerated, use:
6+
// `swift test --generate-linuxmain`
7+
// to regenerate.
8+
static let __allTests__SupportPerfTests = [
9+
("testLineTableAppendPerf", testLineTableAppendPerf),
10+
("testLineTableSingleCharEditPerf", testLineTableSingleCharEditPerf),
11+
]
12+
}
13+
414
extension SupportTests {
515
// DO NOT MODIFY: This is autogenerated, use:
616
// `swift test --generate-linuxmain`
@@ -11,10 +21,8 @@ extension SupportTests {
1121
("testFindSubsequence", testFindSubsequence),
1222
("testIntFromAscii", testIntFromAscii),
1323
("testLineTable", testLineTable),
14-
("testLineTableAppendPerf", testLineTableAppendPerf),
1524
("testLineTableEditing", testLineTableEditing),
1625
("testLineTableLinePositionTranslation", testLineTableLinePositionTranslation),
17-
("testLineTableSingleCharEditPerf", testLineTableSingleCharEditPerf),
1826
("testLogging", testLogging),
1927
("testResultEquality", testResultEquality),
2028
("testResultProjection", testResultProjection),
@@ -23,6 +31,7 @@ extension SupportTests {
2331

2432
public func __allTests() -> [XCTestCaseEntry] {
2533
return [
34+
testCase(SupportPerfTests.__allTests__SupportPerfTests),
2635
testCase(SupportTests.__allTests__SupportTests),
2736
]
2837
}

0 commit comments

Comments
 (0)