Add performance measurement with instructions count

Author: kimdv

.gitignore

@@ -18,3 +18,5 @@ Package.resolved
 
 .DS_Store
 *.pyc
+
+Tests/PerformanceTest/baselines.json

Package.swift

@@ -283,8 +283,8 @@ let package = Package(
 
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
-      exclude: ["Inputs"]
+      dependencies: ["_InstructionCounter", "SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
+      exclude: ["Inputs", "ci-baselines.json"]
     ),
   ]
 )

Tests/PerformanceTest/ParsingPerformanceTests.swift

@@ -25,13 +25,11 @@ public class ParsingPerformanceTests: XCTestCase {
 
   func testNativeParsingPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    measure {
-      do {
-        let source = try String(contentsOf: inputFile)
-        _ = Parser.parse(source: source)
-      } catch {
-        XCTFail(error.localizedDescription)
-      }
+
+    let source = try String(contentsOf: inputFile)
+
+    try measureInstructions {
+      _ = Parser.parse(source: source)
     }
   }
 }

Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift

@@ -26,17 +26,14 @@ public class SyntaxClassifierPerformanceTests: XCTestCase {
 
   func testClassifierPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    XCTAssertNoThrow(
-      try {
-        let source = try String(contentsOf: inputFile)
-        let parsed = Parser.parse(source: source)
 
-        measure {
-          for _ in 0..<10 {
-            for _ in parsed.classifications {}
-          }
-        }
-      }()
-    )
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
+
+    try measureInstructions {
+      for _ in 0..<10 {
+        for _ in parsed.classifications {}
+      }
+    }
   }
 }

Tests/PerformanceTest/VisitorPerformanceTests.swift

@@ -27,50 +27,41 @@ public class VisitorPerformanceTests: XCTestCase {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyVisitor: SyntaxVisitor {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
 
-        let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
+    try measureInstructions {
 
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+      let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
+      emptyVisitor.walk(parsed)
+    }
   }
 
   func testEmptyRewriterPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyRewriter: SyntaxRewriter {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
 
-        let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
+    try measureInstructions {
 
-        measure {
-          _ = emptyRewriter.visit(parsed)
-        }
-      }()
-    )
+      let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
+      _ = emptyRewriter.visit(parsed)
+    }
   }
 
   func testEmptyAnyVisitorPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyAnyVisitor: SyntaxAnyVisitor {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
 
-        let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
+    try measureInstructions {
+      let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
 
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+      emptyVisitor.walk(parsed)
+    }
   }
 }

Tests/PerformanceTest/XCTestCase+InstructionsCount.swift

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import XCTest
+import _InstructionCounter
+
+extension XCTestCase {
+  private var baselineURL: URL {
+    if let baselineFile = ProcessInfo.processInfo.environment["BASELINE_FILE"] {
+      return URL(fileURLWithPath: baselineFile)
+    } else {
+      return URL(fileURLWithPath: #file)
+        .deletingLastPathComponent()
+        .appendingPathComponent("baselines.json")
+    }
+  }
+
+  func measureInstructions(_ baselineName: StaticString = #function, block: () -> Void, file: StaticString = #file, line: UInt = #line) throws {
+    let strippedBaselineName = "\(baselineName)".replacingOccurrences(of: "()", with: "")
+    var baseline: UInt64? = try baseline(for: strippedBaselineName)
+
+    if let environmentBaseline = ProcessInfo.processInfo.environment[strippedBaselineName] {
+      baseline = UInt64(environmentBaseline)
+    }
+
+    let startInstructions = getInstructionsExecuted()
+    block()
+    let endInstructions = getInstructionsExecuted()
+    let numberOfInstructions = endInstructions - startInstructions
+
+    guard let baseline = baseline else {
+      XCTFail("Missing baseline for \(strippedBaselineName) with number of instructions \(numberOfInstructions)", file: file, line: line)
+      return
+    }
+
+    let lowerBaseline = UInt64(Double(baseline) * 0.97)
+    let upperBaseline = UInt64(Double(baseline) * 1.03)
+    let baselineRange = lowerBaseline...upperBaseline
+
+    XCTAssertTrue(
+      baselineRange.contains(numberOfInstructions),
+      """
+      Number of instructions '\(numberOfInstructions)'
+      is not within range \(lowerBaseline)-\(upperBaseline)
+      """,
+      file: file,
+      line: line
+    )
+  }
+
+  private func baseline(for key: String) throws -> UInt64? {
+    let baselineData = try? Data(contentsOf: baselineURL)
+
+    guard let data = baselineData else {
+      return nil
+    }
+
+    let jsonDecoder = JSONDecoder()
+    let baselineMap = try jsonDecoder.decode([String: UInt64].self, from: data)
+
+    return baselineMap[key]
+  }
+}

Tests/PerformanceTest/ci-baselines.json

@@ -0,0 +1,7 @@
+{
+   "testNativeParsingPerformance": 5850081534,
+   "testClassifierPerformance": 27177168966,
+   "testEmptyVisitorPerformance": 6641646112,
+   "testEmptyRewriterPerformance": 6641646112,
+   "testEmptyAnyVisitorPerformance": 6711819049
+}

build-script.py

@@ -16,6 +16,7 @@
 WORKSPACE_DIR = os.path.dirname(PACKAGE_DIR)
 EXAMPLES_DIR = os.path.join(PACKAGE_DIR, "Examples")
 SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources")
+TESTS_DIR = os.path.join(PACKAGE_DIR, "Tests")
 SWIFTIDEUTILS_DIR = os.path.join(SOURCES_DIR, "SwiftIDEUtils")
 SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax")
 SWIFTSYNTAX_DOCUMENTATION_DIR = \
@@ -236,6 +237,7 @@ def __build(self, package_dir: str, name: str, is_product: bool) -> None:
         env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
         env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
             os.path.join(self.toolchain, "lib", "swift", "macosx")
+        env["BASELINE_FILE"] = os.path.join(TESTS_DIR, "PerformanceTest", "ci-baselines.json") 
         check_call(command, env=env, verbose=self.verbose)