Add performance measurement with instructions count

Author: kimdv

.gitignore

@@ -18,3 +18,5 @@ Package.resolved
 
 .DS_Store
 *.pyc
+
+Tests/PerformanceTest/baselines.json

CodeGeneration/Sources/generate-swiftsyntax/LayoutNode+Extensions.swift

@@ -87,13 +87,13 @@ extension LayoutNode {
     }
 
     let formattedParams = """
-      - Parameters:
-        - leadingTrivia: Trivia to be prepended to the leading trivia of the node’s first token. \
-      If the node is empty, there is no token to attach the trivia to and the parameter is ignored.
-      \(children.compactMap(generateParamDocComment).joined(separator: "\n"))
-        - trailingTrivia: Trivia to be appended to the trailing trivia of the node’s last token. \
-      If the node is empty, there is no token to attach the trivia to and the parameter is ignored.
-      """.removingEmptyLines
+    - Parameters:
+      - leadingTrivia: Trivia to be prepended to the leading trivia of the node’s first token. \
+    If the node is empty, there is no token to attach the trivia to and the parameter is ignored.
+    \(children.compactMap(generateParamDocComment).joined(separator: "\n"))
+      - trailingTrivia: Trivia to be appended to the trailing trivia of the node’s last token. \
+    If the node is empty, there is no token to attach the trivia to and the parameter is ignored.
+    """.removingEmptyLines
 
     return docCommentTrivia(from: formattedParams)
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxCollectionsFile.swift

@@ -18,10 +18,10 @@ import Utils
 let syntaxCollectionsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for node in SYNTAX_NODES.compactMap(\.collectionNode) {
     let documentation = """
-      \(node.documentation)
-      \(node.documentation.isEmpty ? "" : "///")
-      \(node.grammar)
-      """.removingEmptyLines
+    \(node.documentation)
+    \(node.documentation.isEmpty ? "" : "///")
+    \(node.grammar)
+    """.removingEmptyLines
 
     try! StructDeclSyntax(
       """

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxNodesFile.swift

@@ -34,10 +34,10 @@ func syntaxNode(emitKind: SyntaxNodeKind) -> SourceFileSyntax {
   SourceFileSyntax(leadingTrivia: copyrightHeader) {
     for node in SYNTAX_NODES.compactMap(\.layoutNode) where node.base == emitKind {
       let documentation = """
-        \(node.documentation)
-        \(node.documentation.isEmpty ? "" : "///")
-        \(node.grammar)
-        """.removingEmptyLines
+      \(node.documentation)
+      \(node.documentation.isEmpty ? "" : "///")
+      \(node.grammar)
+      """.removingEmptyLines
 
       // We are actually handling this node now
       try! StructDeclSyntax(

Package.swift

@@ -283,8 +283,8 @@ let package = Package(
 
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
-      exclude: ["Inputs"]
+      dependencies: ["_InstructionCounter", "SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
+      exclude: ["Inputs", "ci-baselines.json"]
     ),
   ]
 )

Tests/PerformanceTest/ParsingPerformanceTests.swift

@@ -25,7 +25,7 @@ public class ParsingPerformanceTests: XCTestCase {
 
   func testNativeParsingPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    measure {
+    try measureInstructions {
       do {
         let source = try String(contentsOf: inputFile)
         _ = Parser.parse(source: source)

Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift

@@ -26,17 +26,16 @@ public class SyntaxClassifierPerformanceTests: XCTestCase {
 
   func testClassifierPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let source = try String(contentsOf: inputFile)
         let parsed = Parser.parse(source: source)
-
-        measure {
-          for _ in 0..<10 {
-            for _ in parsed.classifications {}
-          }
+        for _ in 0..<10 {
+          for _ in parsed.classifications {}
         }
-      }()
-    )
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 }

Tests/PerformanceTest/VisitorPerformanceTests.swift

@@ -27,50 +27,48 @@ public class VisitorPerformanceTests: XCTestCase {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyVisitor: SyntaxVisitor {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
-
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+        emptyVisitor.walk(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 
   func testEmptyRewriterPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyRewriter: SyntaxRewriter {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
-
-        measure {
-          _ = emptyRewriter.visit(parsed)
-        }
-      }()
-    )
+        _ = emptyRewriter.visit(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 
   func testEmptyAnyVisitorPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyAnyVisitor: SyntaxAnyVisitor {}
 
-    XCTAssertNoThrow(
-      try {
+    try measureInstructions {
+      do {
         let parsed = Parser.parse(source: try String(contentsOf: inputFile))
 
         let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
 
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+        emptyVisitor.walk(parsed)
+      } catch {
+        XCTFail(error.localizedDescription)
+      }
+    }
   }
 }

Tests/PerformanceTest/XCTestCase+InstructionsCount.swift

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import XCTest
+import _InstructionCounter
+
+extension XCTestCase {
+  private var baselineURL: URL {
+    if let baselineFile = ProcessInfo.processInfo.environment["BASELINE_FILE"] {
+      return URL(fileURLWithPath: baselineFile)
+    } else {
+      return URL(fileURLWithPath: #file)
+        .deletingLastPathComponent()
+        .appendingPathComponent("baselines.json")
+    }
+  }
+
+  func measureInstructions(_ baselineName: StaticString = #function, block: () -> Void, file: StaticString = #file, line: UInt = #line) throws {
+    let strippedBaselineName = "\(baselineName)".replacingOccurrences(of: "()", with: "")
+    var baseline: UInt64? = try baseline(for: strippedBaselineName)
+
+    if let environmentBaseline = ProcessInfo.processInfo.environment[strippedBaselineName] {
+      baseline = UInt64(environmentBaseline)
+    }
+
+    guard let baseline = baseline else {
+      XCTFail("Missing baseline for \(strippedBaselineName)", file: file, line: line)
+      return
+    }
+
+    let startInstructions = getInstructionsExecuted()
+    block()
+    let endInstructions = getInstructionsExecuted()
+    let numberOfInstructions = endInstructions - startInstructions
+
+    let lowerBaseline = UInt64(Double(baseline) * 0.97)
+    let upperBaseline = UInt64(Double(baseline) * 1.03)
+    let baselineRange = lowerBaseline...upperBaseline
+
+    XCTAssertTrue(
+      baselineRange.contains(numberOfInstructions),
+      """
+      Number of instructions '\(numberOfInstructions)'
+      is not within range \(lowerBaseline)-\(upperBaseline)
+      """,
+      file: file,
+      line: line
+    )
+  }
+
+  private func baseline(for key: String) throws -> UInt64? {
+    let baselineData = try? Data(contentsOf: baselineURL)
+
+    guard let data = baselineData else {
+      fatalError("No baseline.json provided")
+    }
+
+    let jsonDecoder = JSONDecoder()
+    let baselineMap = try jsonDecoder.decode([String: UInt64].self, from: data)
+
+    return baselineMap[key]
+  }
+}

Tests/PerformanceTest/ci-baselines.json

@@ -0,0 +1,7 @@
+{
+   "testNativeParsingPerformance": 5850081534,
+   "testClassifierPerformance": 27177168966,
+   "testEmptyVisitorPerformance": 6641646112,
+   "testEmptyRewriterPerformance": 6641646112,
+   "testEmptyAnyVisitorPerformance": 6711819049
+}

build-script.py

@@ -16,6 +16,7 @@
 WORKSPACE_DIR = os.path.dirname(PACKAGE_DIR)
 EXAMPLES_DIR = os.path.join(PACKAGE_DIR, "Examples")
 SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources")
+TESTS_DIR = os.path.join(PACKAGE_DIR, "Tests")
 SWIFTIDEUTILS_DIR = os.path.join(SOURCES_DIR, "SwiftIDEUtils")
 SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax")
 SWIFTSYNTAX_DOCUMENTATION_DIR = \
@@ -236,6 +237,7 @@ def __build(self, package_dir: str, name: str, is_product: bool) -> None:
         env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
         env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
             os.path.join(self.toolchain, "lib", "swift", "macosx")
+        env["BASELINE_FILE"] = os.path.join(TESTS_DIR, "PerformanceTest", "ci-baselines.json") 
         check_call(command, env=env, verbose=self.verbose)