Merge pull request #1863 from kimdv/kimdv/add-baseline-for-performance-tests

Author: kimdv

.gitignore

@@ -18,3 +18,5 @@ Package.resolved
 
 .DS_Store
 *.pyc
+
+Tests/PerformanceTest/baselines.json

Package.swift

@@ -283,8 +283,8 @@ let package = Package(
 
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
-      exclude: ["Inputs"]
+      dependencies: ["_InstructionCounter", "SwiftIDEUtils", "SwiftParser", "SwiftSyntax"],
+      exclude: ["Inputs", "ci-baselines.json"]
     ),
   ]
 )

Tests/PerformanceTest/InstructionsCountAssertion.swift

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import XCTest
+import _InstructionCounter
+
+fileprivate var baselineURL: URL {
+  if let baselineFile = ProcessInfo.processInfo.environment["BASELINE_FILE"] {
+    return URL(fileURLWithPath: baselineFile)
+  } else {
+    return URL(fileURLWithPath: #file)
+      .deletingLastPathComponent()
+      .appendingPathComponent("baselines.json")
+  }
+}
+
+func measureInstructions(_ baselineName: StaticString = #function, block: () -> Void, file: StaticString = #file, line: UInt = #line) throws {
+  let startInstructions = getInstructionsExecuted()
+  block()
+  let endInstructions = getInstructionsExecuted()
+  let numberOfInstructions = endInstructions - startInstructions
+  let strippedBaselineName = "\(baselineName)".replacingOccurrences(of: "()", with: "")
+
+  #if os(Darwin)
+  // If the is no data, we just continue the test
+  guard let data = try? Data(contentsOf: baselineURL) else {
+    return
+  }
+
+  let jsonDecoder = JSONDecoder()
+  let baselineMap = try jsonDecoder.decode([String: UInt64].self, from: data)
+
+  guard let baseline = baselineMap[strippedBaselineName] else {
+    XCTFail(
+      """
+      Missing baseline for \(strippedBaselineName)
+      with number of instructions '\(numberOfInstructions)'
+      """,
+      file: file,
+      line: line
+    )
+    return
+  }
+
+  let relativeDeviation = Double(numberOfInstructions) / Double(baseline) - 1
+  let allowedDeviation = 0.01
+
+  XCTAssertTrue(
+    (-allowedDeviation..<allowedDeviation).contains(relativeDeviation),
+    """
+    Number of instructions '\(numberOfInstructions)' deviated from baseline by \(String(format: "%.4f", relativeDeviation * 100))%.
+    The maximum allowed deviation for '\(strippedBaselineName)' is \(allowedDeviation * 100)% in either direction.
+    """,
+    file: file,
+    line: line
+  )
+  #endif
+}

Tests/PerformanceTest/ParsingPerformanceTests.swift

@@ -25,13 +25,11 @@ public class ParsingPerformanceTests: XCTestCase {
 
   func testNativeParsingPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    measure {
-      do {
-        let source = try String(contentsOf: inputFile)
-        _ = Parser.parse(source: source)
-      } catch {
-        XCTFail(error.localizedDescription)
-      }
+
+    let source = try String(contentsOf: inputFile)
+
+    try measureInstructions {
+      _ = Parser.parse(source: source)
     }
   }
 }

Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift

@@ -26,17 +26,14 @@ public class SyntaxClassifierPerformanceTests: XCTestCase {
 
   func testClassifierPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
-    XCTAssertNoThrow(
-      try {
-        let source = try String(contentsOf: inputFile)
-        let parsed = Parser.parse(source: source)
 
-        measure {
-          for _ in 0..<10 {
-            for _ in parsed.classifications {}
-          }
-        }
-      }()
-    )
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
+
+    try measureInstructions {
+      for _ in 0..<10 {
+        for _ in parsed.classifications {}
+      }
+    }
   }
 }

Tests/PerformanceTest/VisitorPerformanceTests.swift

@@ -27,50 +27,38 @@ public class VisitorPerformanceTests: XCTestCase {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyVisitor: SyntaxVisitor {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
+    let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
 
-        let emptyVisitor = EmptyVisitor(viewMode: .sourceAccurate)
-
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+    try measureInstructions {
+      emptyVisitor.walk(parsed)
+    }
   }
 
   func testEmptyRewriterPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyRewriter: SyntaxRewriter {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
-
-        let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
+    let emptyRewriter = EmptyRewriter(viewMode: .sourceAccurate)
 
-        measure {
-          _ = emptyRewriter.visit(parsed)
-        }
-      }()
-    )
+    try measureInstructions {
+      _ = emptyRewriter.visit(parsed)
+    }
   }
 
   func testEmptyAnyVisitorPerformance() throws {
     try XCTSkipIf(ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] == "1")
     class EmptyAnyVisitor: SyntaxAnyVisitor {}
 
-    XCTAssertNoThrow(
-      try {
-        let parsed = Parser.parse(source: try String(contentsOf: inputFile))
-
-        let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
+    let source = try String(contentsOf: inputFile)
+    let parsed = Parser.parse(source: source)
+    let emptyVisitor = EmptyAnyVisitor(viewMode: .sourceAccurate)
 
-        measure {
-          emptyVisitor.walk(parsed)
-        }
-      }()
-    )
+    try measureInstructions {
+      emptyVisitor.walk(parsed)
+    }
   }
 }

Tests/PerformanceTest/ci-baselines.json

@@ -0,0 +1,7 @@
+{
+   "testNativeParsingPerformance": 307024328,
+   "testClassifierPerformance": 2624475912,
+   "testEmptyVisitorPerformance": 48502831,
+   "testEmptyRewriterPerformance": 54438776,
+   "testEmptyAnyVisitorPerformance": 49123765
+}

build-script.py

@@ -16,6 +16,7 @@
 WORKSPACE_DIR = os.path.dirname(PACKAGE_DIR)
 EXAMPLES_DIR = os.path.join(PACKAGE_DIR, "Examples")
 SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources")
+TESTS_DIR = os.path.join(PACKAGE_DIR, "Tests")
 SWIFTIDEUTILS_DIR = os.path.join(SOURCES_DIR, "SwiftIDEUtils")
 SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax")
 SWIFTSYNTAX_DOCUMENTATION_DIR = \
@@ -236,6 +237,7 @@ def __build(self, package_dir: str, name: str, is_product: bool) -> None:
         env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
         env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
             os.path.join(self.toolchain, "lib", "swift", "macosx")
+
         check_call(command, env=env, verbose=self.verbose)
 
 
@@ -463,6 +465,8 @@ def run_xctests(
     env["SWIFTCI_USE_LOCAL_DEPS"] = "1"
     env["SWIFT_SYNTAX_PARSER_LIB_SEARCH_PATH"] = \
         os.path.join(toolchain, "lib", "swift", "macosx")
+    env["BASELINE_FILE"] = os.path.join(TESTS_DIR, "PerformanceTest", "ci-baselines.json") 
+
     check_call(swiftpm_call, env=env, verbose=verbose)
 
 # -----------------------------------------------------------------------------