Add performance measurement with instructions count

Author: kimdv

.gitignore

@@ -18,3 +18,5 @@ Package.resolved
 
 .DS_Store
 *.pyc
+
+Tests/PerformanceTest/baselines.json

Package.swift

@@ -283,8 +283,8 @@ let package = Package(
 
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
-      exclude: ["Inputs"]
+      dependencies: ["_InstructionCounter", "SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
+      exclude: ["Inputs", "ci-baselines.json"]
     ),
   ]
 )

Tests/PerformanceTest/ParsingPerformanceTests.swift

@@ -25,7 +25,7 @@ public class ParsingPerformanceTests: XCTestCase {
 
   func testNativeParsingPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    measure {
+    try measureInstructions {
       do {
         let source = try String(contentsOf: inputFile)
         _ = Parser.parse(source: source)

Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift

@@ -26,17 +26,16 @@ public class SyntaxClassifierPerformanceTests: XCTestCase {
 
   func testClassifierPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let source = try String(contentsOf: inputFile)
         let parsed = Parser.parse(source: source)
-
-        measure {
-          for _ in 0..<10 {
-            for _ in parsed.classifications {}
-          }
+        for _ in 0..<10 {
+          for _ in parsed.classifications {}
         }
-      }()
-    )
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 }

Tests/PerformanceTest/VisitorPerformanceTests.swift

@@ -27,50 +27,48 @@ public class VisitorPerformanceTests: XCTestCase {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyVisitor: SyntaxVisitor {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
-
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+        emptyVisitor.walk(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 
   func testEmptyRewriterPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyRewriter: SyntaxRewriter {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
-
-        measure {
-          _ = emptyRewriter.visit(parsed)
-        }
-      }()
-    )
+        _ = emptyRewriter.visit(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 
   func testEmptyAnyVisitorPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyAnyVisitor: SyntaxAnyVisitor {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
 
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+        emptyVisitor.walk(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 }

Tests/PerformanceTest/XCTestCase+InstructionsCount.swift

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import XCTest
+import _InstructionCounter
+
+extension XCTestCase {
+  private var baselineURL: URL {
+    if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == "1" {
+      return URL(fileURLWithPath: #file)
+        .deletingLastPathComponent()
+        .appendingPathComponent("ci-baselines.json")
+    } else {
+      return URL(fileURLWithPath: #file)
+        .deletingLastPathComponent()
+        .appendingPathComponent("baselines.json")
+    }
+  }
+
+  func measureInstructions(_ baselineName: StaticString = #function, block: () -> Void, file: StaticString = #file, line: UInt = #line) throws {
+    let strippedBaselineName = "\(baselineName)".replacingOccurrences(of: "()", with: "")
+    var baseline: UInt64? = try baseline(for: strippedBaselineName)
+
+    if let environmentBaseline = ProcessInfo.processInfo.environment[strippedBaselineName] {
+      baseline = UInt64(environmentBaseline)
+    }
+
+    guard let baseline = baseline else {
+      XCTFail("Missing baseline for \(strippedBaselineName)", file: file, line: line)
+      return
+    }
+
+    let startInstructions = getInstructionsExecuted()
+    block()
+    let endInstructions = getInstructionsExecuted()
+    let numberOfInstructions = endInstructions - startInstructions
+
+    let lowerBaseline = UInt64(Double(baseline) * 0.97)
+    let upperBaseline = UInt64(Double(baseline) * 1.03)
+    let baselineRange = lowerBaseline...upperBaseline
+
+    XCTAssertTrue(
+      baselineRange.contains(numberOfInstructions),
+      """
+      Number of instructions '\(numberOfInstructions)'
+      is not within range \(lowerBaseline)-\(upperBaseline)
+      """,
+      file: file,
+      line: line
+    )
+  }
+
+  private func baseline(for key: String) throws -> UInt64? {
+    let baselineData = try? Data(contentsOf: baselineURL)
+
+    guard let data = baselineData else {
+      fatalError("No baseline.json provided")
+    }
+
+    let jsonDecoder = JSONDecoder()
+    let baselineMap = try jsonDecoder.decode([String: UInt64].self, from: data)
+
+    return baselineMap[key]
+  }
+}

Tests/PerformanceTest/ci-baselines.json

@@ -0,0 +1,7 @@
+{
+   "testNativeParsingPerformance": 5850081534,
+   "testClassifierPerformance": 27177168966,
+   "testEmptyVisitorPerformance": 6641646112,
+   "testEmptyRewriterPerformance": 6641646112,
+   "testEmptyAnyVisitorPerformance": 6711819049
+}