Skip to content

Use Swift Concurrency in SwiftcImportScanner #6996

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 12, 2023
Merged

Use Swift Concurrency in SwiftcImportScanner #6996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf7b2c6
Use Swift Concurrency in `SwiftcImportScanner`
MaxDesiatov Oct 12, 2023
0cfd8e8
Apply `callbackQueue` argument in `validateImports`
MaxDesiatov Oct 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 6 additions & 27 deletions Sources/Basics/ImportScanning.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@ private struct Imports: Decodable {
}

public protocol ImportScanner {
func scanImports(
_ filePathToScan: AbsolutePath,
callbackQueue: DispatchQueue,
completion: @escaping (Result<[String], Error>) -> Void
)
func scanImports(_ filePathToScan: AbsolutePath) async throws -> [String]
}

public struct SwiftcImportScanner: ImportScanner {
Expand All @@ -45,32 +41,15 @@ public struct SwiftcImportScanner: ImportScanner {
self.swiftCompilerPath = swiftCompilerPath
}

public func scanImports(
_ filePathToScan: AbsolutePath,
callbackQueue: DispatchQueue,
completion: @escaping (Result<[String], Error>) -> Void
) {
public func scanImports(_ filePathToScan: AbsolutePath) async throws -> [String] {
let cmd = [swiftCompilerPath.pathString,
filePathToScan.pathString,
"-scan-dependencies", "-Xfrontend", "-import-prescan"] + self.swiftCompilerFlags

TSCBasic.Process
.popen(arguments: cmd, environment: self.swiftCompilerEnvironment, queue: callbackQueue) { result in
dispatchPrecondition(condition: .onQueue(callbackQueue))

do {
let stdout = try result.get().utf8Output()
let imports = try JSONDecoder.makeWithDefaults().decode(Imports.self, from: stdout).imports
.filter { !defaultImports.contains($0) }
let result = try await TSCBasic.Process.popen(arguments: cmd, environment: self.swiftCompilerEnvironment)

callbackQueue.async {
completion(.success(imports))
}
} catch {
callbackQueue.async {
completion(.failure(error))
}
}
}
let stdout = try result.utf8Output()
return try JSONDecoder.makeWithDefaults().decode(Imports.self, from: stdout).imports
.filter { !defaultImports.contains($0) }
}
}
27 changes: 13 additions & 14 deletions Sources/PackageLoading/ManifestLoader.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
//
//===----------------------------------------------------------------------===//

import _Concurrency
import Basics
import Dispatch
@_implementationOnly import Foundation
Expand Down Expand Up @@ -658,23 +659,21 @@ public final class ManifestLoader: ManifestLoaderProtocol {

// we must not block the calling thread (for concurrency control) so nesting this in a queue
self.evaluationQueue.addOperation {
do {
// park the evaluation thread based on the max concurrency allowed
self.concurrencySemaphore.wait()
// park the evaluation thread based on the max concurrency allowed
self.concurrencySemaphore.wait()

let importScanner = SwiftcImportScanner(swiftCompilerEnvironment: self.toolchain.swiftCompilerEnvironment,
swiftCompilerFlags: self.extraManifestFlags,
swiftCompilerPath: self.toolchain.swiftCompilerPathForManifests)
let importScanner = SwiftcImportScanner(swiftCompilerEnvironment: self.toolchain.swiftCompilerEnvironment,
swiftCompilerFlags: self.extraManifestFlags,
swiftCompilerPath: self.toolchain.swiftCompilerPathForManifests)

importScanner.scanImports(manifestPath, callbackQueue: callbackQueue) { result in
do {
let imports = try result.get().filter { !allowedImports.contains($0) }
guard imports.isEmpty else {
throw ManifestParseError.importsRestrictedModules(imports)
}
} catch {
completion(.failure(error))
Task {
let result = try await importScanner.scanImports(manifestPath)
let imports = result.filter { !allowedImports.contains($0) }
guard imports.isEmpty else {
callbackQueue.async {
completion(.failure(ManifestParseError.importsRestrictedModules(imports)))
}
return
}
}
}
Expand Down
21 changes: 21 additions & 0 deletions Sources/SPMTestSupport/misc.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,40 @@ import enum TSCUtility.Git
@_exported import func TSCTestSupport.systemQuietly
@_exported import enum TSCTestSupport.StringPattern

/// Test helper utility for executing a block with a temporary directory.
public func testWithTemporaryDirectory(
function: StaticString = #function,
body: (AbsolutePath) throws -> Void
) throws {
let body2 = { (path: TSCAbsolutePath) in
try body(AbsolutePath(path))
}

try TSCTestSupport.testWithTemporaryDirectory(
function: function,
body: body2
)
}

public func testWithTemporaryDirectory(
function: StaticString = #function,
body: (AbsolutePath) async throws -> Void
) async throws {
let cleanedFunction = function.description
.replacingOccurrences(of: "(", with: "")
.replacingOccurrences(of: ")", with: "")
.replacingOccurrences(of: ".", with: "")
.replacingOccurrences(of: ":", with: "_")
try await withTemporaryDirectory(prefix: "spm-tests-\(cleanedFunction)") { tmpDirPath in
defer {
// Unblock and remove the tmp dir on deinit.
try? localFileSystem.chmod(.userWritable, path: tmpDirPath, options: [.recursive])
try? localFileSystem.removeFileTree(tmpDirPath)
}
try await body(tmpDirPath)
}
}

/// Test-helper function that runs a block of code on a copy of a test fixture
/// package. The copy is made into a temporary directory, and the block is
/// given a path to that directory. The block is permitted to modify the copy.
Expand Down
16 changes: 5 additions & 11 deletions Sources/Workspace/Workspace.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1409,8 +1409,8 @@ extension Workspace {
}

public func loadPluginImports(
packageGraph: PackageGraph,
completion: @escaping(Result<[PackageIdentity: [String: [String]]], Error>) -> Void) {
packageGraph: PackageGraph
) async throws -> [PackageIdentity: [String: [String]]] {
let pluginTargets = packageGraph.allTargets.filter{$0.type == .plugin}
let scanner = SwiftcImportScanner(swiftCompilerEnvironment: hostToolchain.swiftCompilerEnvironment, swiftCompilerFlags: hostToolchain.swiftCompilerFlags + ["-I", hostToolchain.swiftPMLibrariesLocation.pluginLibraryPath.pathString], swiftCompilerPath: hostToolchain.swiftCompilerPath)
var importList = [PackageIdentity: [String: [String]]]()
Expand All @@ -1426,17 +1426,11 @@ extension Workspace {
}

for path in paths {
do {
let result = try temp_await {
scanner.scanImports(path, callbackQueue: DispatchQueue.sharedConcurrent, completion: $0)
}
importList[pkgId]?[pluginTarget.name]?.append(contentsOf: result)
} catch {
completion(.failure(error))
}
let result = try await scanner.scanImports(path)
importList[pkgId]?[pluginTarget.name]?.append(contentsOf: result)
}
}
completion(.success(importList))
return importList
}

/// Loads a single package in the context of a previously loaded graph. This can be useful for incremental loading in a longer-lived program, like an IDE.
Expand Down
62 changes: 29 additions & 33 deletions Tests/SPMBuildCoreTests/PluginInvocationTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -916,11 +916,11 @@ class PluginInvocationTests: XCTestCase {
}
}

func testScanImportsInPluginTargets() throws {
func testScanImportsInPluginTargets() async throws {
// Only run the test if the environment in which we're running actually supports Swift concurrency (which the plugin APIs require).
try XCTSkipIf(!UserToolchain.default.supportsSwiftConcurrency(), "skipping because test environment doesn't support concurrency")

try testWithTemporaryDirectory { tmpPath in
try await testWithTemporaryDirectory { tmpPath in
// Create a sample package with a library target and a plugin.
let packageDir = tmpPath.appending(components: "MyPackage")
try localFileSystem.createDirectory(packageDir, recursive: true)
Expand Down Expand Up @@ -1058,39 +1058,35 @@ class PluginInvocationTests: XCTestCase {
XCTAssert(rootManifests.count == 1, "\(rootManifests)")

let graph = try workspace.loadPackageGraph(rootInput: rootInput, observabilityScope: observability.topScope)
workspace.loadPluginImports(packageGraph: graph) { (result: Result<[PackageIdentity : [String : [String]]], Error>) in

var count = 0
if let dict = try? result.get() {
for (pkg, entry) in dict {
if pkg.description == "mypackage" {
XCTAssertNotNil(entry["XPlugin"])
let XPluginPossibleImports1 = ["PackagePlugin", "XcodeProjectPlugin"]
let XPluginPossibleImports2 = ["PackagePlugin", "XcodeProjectPlugin", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["XPlugin"] == XPluginPossibleImports1 ||
entry["XPlugin"] == XPluginPossibleImports2)

let YPluginPossibleImports1 = ["PackagePlugin", "Foundation"]
let YPluginPossibleImports2 = ["PackagePlugin", "Foundation", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["YPlugin"] == YPluginPossibleImports1 ||
entry["YPlugin"] == YPluginPossibleImports2)
count += 1
} else if pkg.description == "otherpackage" {
XCTAssertNotNil(dict[pkg]?["QPlugin"])

let possibleImports1 = ["PackagePlugin", "XcodeProjectPlugin", "ModuleFoundViaExtraSearchPaths"]
let possibleImports2 = ["PackagePlugin", "XcodeProjectPlugin", "ModuleFoundViaExtraSearchPaths", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["QPlugin"] == possibleImports1 ||
entry["QPlugin"] == possibleImports2)
count += 1
}
}
} else {
XCTFail("Scanned import list should not be empty")
let dict = try await workspace.loadPluginImports(packageGraph: graph)

var count = 0
for (pkg, entry) in dict {
if pkg.description == "mypackage" {
XCTAssertNotNil(entry["XPlugin"])
let XPluginPossibleImports1 = ["PackagePlugin", "XcodeProjectPlugin"]
let XPluginPossibleImports2 = ["PackagePlugin", "XcodeProjectPlugin", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["XPlugin"] == XPluginPossibleImports1 ||
entry["XPlugin"] == XPluginPossibleImports2)

let YPluginPossibleImports1 = ["PackagePlugin", "Foundation"]
let YPluginPossibleImports2 = ["PackagePlugin", "Foundation", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["YPlugin"] == YPluginPossibleImports1 ||
entry["YPlugin"] == YPluginPossibleImports2)
count += 1
} else if pkg.description == "otherpackage" {
XCTAssertNotNil(dict[pkg]?["QPlugin"])

let possibleImports1 = ["PackagePlugin", "XcodeProjectPlugin", "ModuleFoundViaExtraSearchPaths"]
let possibleImports2 = ["PackagePlugin", "XcodeProjectPlugin", "ModuleFoundViaExtraSearchPaths", "_SwiftConcurrencyShims"]
XCTAssertTrue(entry["QPlugin"] == possibleImports1 ||
entry["QPlugin"] == possibleImports2)
count += 1
}

XCTAssertEqual(count, 2)
}


XCTAssertEqual(count, 2)
}
}

Expand Down