Merge main into release/6.2

.github/workflows/pull_request.yml

@@ -14,11 +14,52 @@ jobs:
     uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
     with:
       linux_os_versions: '["noble", "jammy", "focal", "rhel-ubi9"]'
-      linux_pre_build_command: command -v apt >/dev/null 2>&1 && apt update && apt install -y libsqlite3-dev libncurses-dev || (command -v yum >/dev/null 2>&1 && yum update -y && yum install -y sqlite-devel ncurses-devel)
+      linux_pre_build_command: |
+        if command -v apt-get >/dev/null 2>&1 ; then # bookworm, noble, jammy, focal
+          apt-get update -y
+
+          # Build dependencies
+          apt-get install -y libsqlite3-dev libncurses-dev
+
+          # Debug symbols
+          apt-get install -y libc6-dbg
+        elif command -v dnf >/dev/null 2>&1 ; then # rhel-ubi9
+          dnf update -y
+
+          # Build dependencies
+          dnf install -y sqlite-devel ncurses-devel
+
+          # Debug symbols
+          dnf debuginfo-install -y glibc
+        elif command -v yum >/dev/null 2>&1 ; then # amazonlinux2
+          yum update -y
+
+          # Build dependencies
+          yum install -y sqlite-devel ncurses-devel
+
+          # Debug symbols
+          yum install -y yum-utils
+          debuginfo-install -y glibc
+        fi
       linux_build_command: 'swift test --no-parallel'
       linux_swift_versions: '["nightly-main", "nightly-6.2"]'
       windows_swift_versions: '["nightly-main"]'
       windows_build_command: 'swift test --no-parallel'
+  cmake-smoke-test:
+    name: cmake-smoke-test
+    uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
+    with:
+      linux_os_versions: '["noble"]'
+      linux_pre_build_command: |
+        apt-get update -y
+
+        # Build dependencies
+        apt-get install -y libsqlite3-dev libncurses-dev
+
+        apt-get install -y cmake ninja-build
+      linux_build_command: 'swift package -Xbuild-tools-swiftc -DUSE_PROCESS_SPAWNING_WORKAROUND cmake-smoke-test --disable-sandbox --cmake-path `which cmake` --ninja-path `which ninja` --extra-cmake-arg -DCMAKE_C_COMPILER=`which clang` --extra-cmake-arg -DCMAKE_CXX_COMPILER=`which clang++` --extra-cmake-arg -DCMAKE_Swift_COMPILER=`which swiftc`'
+      linux_swift_versions: '["nightly-main"]'
+      windows_swift_versions: '[]'
   soundness:
     name: Soundness
     uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main

Plugins/cmake-smoke-test/cmake-smoke-test.swift

@@ -24,21 +24,24 @@ struct CMakeSmokeTest: CommandPlugin {
         }
 
         guard let cmakePath = args.extractOption(named: "cmake-path").last else { throw Errors.missingRequiredOption("--cmake-path") }
-        print("using cmake at \(cmakePath)")
+        Diagnostics.progress("using cmake at \(cmakePath)")
         let cmakeURL = URL(filePath: cmakePath)
         guard let ninjaPath = args.extractOption(named: "ninja-path").last else { throw Errors.missingRequiredOption("--ninja-path") }
-        print("using ninja at \(ninjaPath)")
+        Diagnostics.progress("using ninja at \(ninjaPath)")
         let ninjaURL = URL(filePath: ninjaPath)
         let sysrootPath = args.extractOption(named: "sysroot-path").last
         if let sysrootPath {
-            print("using sysroot at \(sysrootPath)")
+            Diagnostics.progress("using sysroot at \(sysrootPath)")
         }
 
+        let extraCMakeArgs = args.extractOption(named: "extra-cmake-arg")
+        Diagnostics.progress("Extra cmake args: \(extraCMakeArgs.joined(separator: " "))")
+
         let moduleCachePath = context.pluginWorkDirectoryURL.appending(component: "module-cache").path()
 
         let swiftBuildURL = context.package.directoryURL
         let swiftBuildBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-build")
-        print("swift-build: \(swiftBuildURL.path())")
+        Diagnostics.progress("swift-build: \(swiftBuildURL.path())")
 
         let swiftToolsSupportCoreURL = try findDependency("swift-tools-support-core", pluginContext: context)
         let swiftToolsSupportCoreBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-tools-support-core")
@@ -80,39 +83,39 @@ struct CMakeSmokeTest: CommandPlugin {
             "-DCMAKE_MAKE_PROGRAM=\(ninjaPath)",
             "-DCMAKE_BUILD_TYPE:=Debug",
             "-DCMAKE_Swift_FLAGS='\(sharedSwiftFlags.joined(separator: " "))'"
-        ] + cMakeProjectArgs
+        ] + cMakeProjectArgs + extraCMakeArgs
 
-        print("Building swift-tools-support-core")
+        Diagnostics.progress("Building swift-tools-support-core")
         try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftToolsSupportCoreURL.path()], workingDirectory: swiftToolsSupportCoreBuildURL)
         try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftToolsSupportCoreBuildURL)
-        print("Built swift-tools-support-core")
+        Diagnostics.progress("Built swift-tools-support-core")
 
         if hostOS != .macOS {
-            print("Building swift-system")
+            Diagnostics.progress("Building swift-system")
             try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftSystemURL.path()], workingDirectory: swiftSystemBuildURL)
             try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftSystemBuildURL)
-            print("Built swift-system")
+            Diagnostics.progress("Built swift-system")
         }
 
-        print("Building llbuild")
+        Diagnostics.progress("Building llbuild")
         try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + ["-DLLBUILD_SUPPORT_BINDINGS:=Swift", llbuildURL.path()], workingDirectory: llbuildBuildURL)
         try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: llbuildBuildURL)
-        print("Built llbuild")
+        Diagnostics.progress("Built llbuild")
 
-        print("Building swift-argument-parser")
+        Diagnostics.progress("Building swift-argument-parser")
         try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + ["-DBUILD_TESTING=NO", "-DBUILD_EXAMPLES=NO", swiftArgumentParserURL.path()], workingDirectory: swiftArgumentParserBuildURL)
         try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftArgumentParserBuildURL)
-        print("Built swift-argument-parser")
+        Diagnostics.progress("Built swift-argument-parser")
 
-        print("Building swift-driver")
+        Diagnostics.progress("Building swift-driver")
         try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftDriverURL.path()], workingDirectory: swiftDriverBuildURL)
         try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftDriverBuildURL)
-        print("Built swift-driver")
+        Diagnostics.progress("Built swift-driver")
 
-        print("Building swift-build in \(swiftBuildBuildURL)")
+        Diagnostics.progress("Building swift-build in \(swiftBuildBuildURL)")
         try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftBuildURL.path()], workingDirectory: swiftBuildBuildURL)
         try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftBuildBuildURL)
-        print("Built swift-build")
+        Diagnostics.progress("Built swift-build")
     }
 
     func findDependency(_ name: String, pluginContext: PluginContext) throws -> URL {
@@ -132,7 +135,7 @@ struct CMakeSmokeTest: CommandPlugin {
             throw Errors.missingRepository(name)
         }
         let dependencyURL = dependency.directoryURL
-        print("\(name): \(dependencyURL.path())")
+        Diagnostics.progress("\(name): \(dependencyURL.path())")
         guard FileManager.default.fileExists(atPath: dependencyURL.path()) else {
             throw Errors.missingRepository(dependencyURL.path())
         }
@@ -145,6 +148,7 @@ enum Errors: Error {
     case missingRequiredOption(String)
     case missingRepository(String)
     case unimplementedForHostOS
+    case miscError(String)
 }
 
 enum OS {
@@ -182,7 +186,53 @@ extension Process {
     }
 
     static func checkNonZeroExit(url: URL, arguments: [String], workingDirectory: URL, environment: [String: String]? = nil) async throws {
-        print("\(url.path()) \(arguments.joined(separator: " "))")
+        Diagnostics.progress("\(url.path()) \(arguments.joined(separator: " "))")
+        #if USE_PROCESS_SPAWNING_WORKAROUND
+        Diagnostics.progress("Using process spawning workaround")
+        // Linux workaround for https://github.com/swiftlang/swift-corelibs-foundation/issues/4772
+        // Foundation.Process on Linux seems to inherit the Process.run()-calling thread's signal mask, creating processes that even have SIGTERM blocked
+        // This manifests as CMake hanging when invoking 'uname' with incorrectly configured signal handlers.
+        var fileActions = posix_spawn_file_actions_t()
+        defer { posix_spawn_file_actions_destroy(&fileActions) }
+        var attrs: posix_spawnattr_t = posix_spawnattr_t()
+        defer { posix_spawnattr_destroy(&attrs) }
+        posix_spawn_file_actions_init(&fileActions)
+        posix_spawn_file_actions_addchdir_np(&fileActions, workingDirectory.path())
+
+        posix_spawnattr_init(&attrs)
+        posix_spawnattr_setpgroup(&attrs, 0)
+        var noSignals = sigset_t()
+        sigemptyset(&noSignals)
+        posix_spawnattr_setsigmask(&attrs, &noSignals)
+
+        var mostSignals = sigset_t()
+        sigemptyset(&mostSignals)
+        for i in 1 ..< SIGSYS {
+            if i == SIGKILL || i == SIGSTOP {
+                continue
+            }
+            sigaddset(&mostSignals, i)
+        }
+        posix_spawnattr_setsigdefault(&attrs, &mostSignals)
+        posix_spawnattr_setflags(&attrs, numericCast(POSIX_SPAWN_SETPGROUP | POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK))
+        var pid: pid_t = -1
+        try withArrayOfCStrings([url.path()] + arguments) { arguments in
+            try withArrayOfCStrings((environment ?? [:]).map { key, value in "\(key)=\(value)" }) { environment in
+                let spawnResult = posix_spawn(&pid, url.path(), /*file_actions=*/&fileActions, /*attrp=*/&attrs, arguments, nil);
+                var exitCode: Int32 = -1
+                var result = wait4(pid, &exitCode, 0, nil);
+                while (result == -1 && errno == EINTR) {
+                    result = wait4(pid, &exitCode, 0, nil)
+                }
+                guard result != -1 else {
+                    throw Errors.miscError("wait failed")
+                }
+                guard exitCode == 0 else {
+                    throw Errors.miscError("exit code nonzero")
+                }
+            }
+        }
+        #else
         let process = Process()
         process.executableURL = url
         process.arguments = arguments
@@ -192,5 +242,44 @@ extension Process {
         if process.terminationStatus != 0 {
             throw Errors.processError(terminationReason: process.terminationReason, terminationStatus: process.terminationStatus)
         }
+        #endif
+    }
+}
+
+func scan<S: Sequence, U>(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] {
+  var result: [U] = []
+  result.reserveCapacity(seq.underestimatedCount)
+  var runningResult = initial
+  for element in seq {
+    runningResult = combine(runningResult, element)
+    result.append(runningResult)
+  }
+  return result
+}
+
+func withArrayOfCStrings<T>(
+  _ args: [String],
+  _ body: (UnsafePointer<UnsafeMutablePointer<Int8>?>) throws -> T
+) throws -> T {
+  let argsCounts = Array(args.map { $0.utf8.count + 1 })
+  let argsOffsets = [0] + scan(argsCounts, 0, +)
+  let argsBufferSize = argsOffsets.last!
+  var argsBuffer: [UInt8] = []
+  argsBuffer.reserveCapacity(argsBufferSize)
+  for arg in args {
+    argsBuffer.append(contentsOf: arg.utf8)
+    argsBuffer.append(0)
+  }
+  return try argsBuffer.withUnsafeMutableBufferPointer {
+    (argsBuffer) in
+    let ptr = UnsafeRawPointer(argsBuffer.baseAddress!).bindMemory(
+      to: Int8.self, capacity: argsBuffer.count)
+    var cStrings: [UnsafePointer<Int8>?] = argsOffsets.map { ptr + $0 }
+    cStrings[cStrings.count - 1] = nil
+    return try cStrings.withUnsafeMutableBufferPointer {
+      let unsafeString = UnsafeMutableRawPointer($0.baseAddress!).bindMemory(
+        to: UnsafeMutablePointer<Int8>?.self, capacity: $0.count)
+      return try body(unsafeString)
     }
+  }
 }

Sources/SWBBuildService/BuildDependencyInfo.swift

@@ -117,6 +117,7 @@ package struct BuildDependencyInfo: Codable {
             package enum LibraryType: String, Codable, Sendable {
                 case dynamic
                 case `static`
+                case upward
                 case unknown
             }
 
@@ -434,9 +435,11 @@ extension BuildDependencyInfo {
     /// - remark: This is written somewhat generically (with the callback blocks) in the hopes that `LinkageDependencyResolver.dependencies(for:...)` can someday adopt it, as the general approach was stolen from there.
     package static func findLinkedInputsFromBuildSettings(_ settings: Settings, addFramework: @Sendable (TargetDependencyInfo.Input) async -> Void, addLibrary: @Sendable (TargetDependencyInfo.Input) async -> Void, addError: @Sendable (String) async -> Void) async {
         await LdLinkerSpec.processLinkerSettingsForLibraryOptions(settings: settings) { macro, flag, stem in
-            await addFramework(TargetDependencyInfo.Input(inputType: .framework, name: .stem(stem), linkType: .searchPath, libraryType: .dynamic))
+            let libType: TargetDependencyInfo.Input.LibraryType = (flag == "-upward_framework") ? .upward : .dynamic
+            await addFramework(TargetDependencyInfo.Input(inputType: .framework, name: .stem(stem), linkType: .searchPath, libraryType: libType))
         } addLibrary: { macro, flag, stem in
-            await addLibrary(TargetDependencyInfo.Input(inputType: .library, name: .stem(stem), linkType: .searchPath, libraryType: .unknown))
+            let libType: TargetDependencyInfo.Input.LibraryType = (flag == "-upward-l") ? .upward : .unknown
+            await addLibrary(TargetDependencyInfo.Input(inputType: .library, name: .stem(stem), linkType: .searchPath, libraryType: libType))
         } addError: { error in
             await addError(error)
         }

Sources/SWBBuildSystem/BuildOperation.swift

@@ -419,7 +419,7 @@ package final class BuildOperation: BuildSystemOperation {
         }
 
         // Perform any needed steps before we kick off the build.
-        if let (warnings, errors) = prepareForBuilding() {
+        if let (warnings, errors) = await prepareForBuilding() {
             // Emit any warnings and errors.  If there were any errors, then bail out.
             for message in warnings { buildOutputDelegate.warning(message) }
             for message in errors { buildOutputDelegate.error(message) }
@@ -809,7 +809,7 @@ package final class BuildOperation: BuildSystemOperation {
         return delegate.buildComplete(self, status: effectiveStatus, delegate: buildOutputDelegate, metrics: .init(counters: aggregatedCounters))
     }
 
-    func prepareForBuilding() -> ([String], [String])? {
+    func prepareForBuilding() async -> ([String], [String])? {
         let warnings = [String]()       // Not presently used
         var errors = [String]()
 
@@ -829,9 +829,61 @@ package final class BuildOperation: BuildSystemOperation {
             }
         }
 
+        if UserDefaults.enableCASValidation {
+            for info in buildDescription.casValidationInfos {
+                do {
+                    try await validateCAS(info)
+                } catch {
+                    errors.append("cas validation failed for \(info.options.casPath.str)")
+                }
+            }
+        }
+
         return (warnings.count > 0 || errors.count > 0) ? (warnings, errors) : nil
     }
 
+    func validateCAS(_ info: BuildDescription.CASValidationInfo) async throws {
+        assert(UserDefaults.enableCASValidation)
+
+        let casPath = info.options.casPath
+        let ruleInfo = "ValidateCAS \(casPath.str) \(info.llvmCasExec.str)"
+
+        let signatureCtx = InsecureHashContext()
+        signatureCtx.add(string: "ValidateCAS")
+        signatureCtx.add(string: casPath.str)
+        signatureCtx.add(string: info.llvmCasExec.str)
+        let signature = signatureCtx.signature
+
+        let activityId = delegate.beginActivity(self, ruleInfo: ruleInfo, executionDescription: "Validate CAS contents at \(casPath.str)", signature: signature, target: nil, parentActivity: nil)
+        var status: BuildOperationTaskEnded.Status = .failed
+        defer {
+            delegate.endActivity(self, id: activityId, signature: signature, status: status)
+        }
+
+        var commandLine = [
+            info.llvmCasExec.str,
+            "-cas", casPath.str,
+            "-validate-if-needed",
+            "-check-hash",
+            "-allow-recovery",
+        ]
+        if let pluginPath = info.options.pluginPath {
+            commandLine.append(contentsOf: [
+                "-fcas-plugin-path", pluginPath.str
+            ])
+        }
+        let result: Processes.ExecutionResult = try await clientDelegate.executeExternalTool(commandLine: commandLine)
+        // In a task we might use a discovered tool info to detect if the tool supports validation, but without that scaffolding, just check the specific error.
+        if result.exitStatus == .exit(1) && result.stderr.contains(ByteString("Unknown command line argument '-validate-if-needed'")) {
+            delegate.emit(data: ByteString("validation not supported").bytes, for: activityId, signature: signature)
+            status = .succeeded
+        } else {
+            delegate.emit(data: ByteString(result.stderr).bytes, for: activityId, signature: signature)
+            delegate.emit(data: ByteString(result.stdout).bytes, for: activityId, signature: signature)
+            status = result.exitStatus.isSuccess ? .succeeded : result.exitStatus.wasCanceled ? .cancelled : .failed
+        }
+    }
+
     /// Cancel the executing build operation.
     package func cancel() {
         queue.blocking_sync() {
@@ -1850,15 +1902,19 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act
             return
         }
 
+        guard let outputDelegate = (queue.blocking_sync { self.commandOutputDelegates.removeValue(forKey: command) }) else {
+            // If there's no outputDelegate, the command never started (i.e. it was skipped by shouldCommandStart().
+            return
+        }
+
+        // We can call this here because we're on an llbuild worker thread. This shouldn't be used while on `self.queue` because we have Swift async work elsewhere which blocks on that queue.
+        let sandboxViolations = task.isSandboxed && result == .failed ? task.extractSandboxViolationMessages_ASYNC_UNSAFE(startTime: outputDelegate.startTime) : []
+
         queue.async {
-            // Find the output delegate, and remove it from the active set.
-            guard let outputDelegate = self.commandOutputDelegates.removeValue(forKey: command) else {
-                // If there's no outputDelegate, the command never started (i.e. it was skipped by shouldCommandStart().
-                return
+            for message in sandboxViolations {
+                outputDelegate.emit(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData(message)))
             }
 
-            outputDelegate.emitSandboxingViolations(task: task, commandResult: result)
-
             // This may be updated by commandProcessFinished if it was an
             // ExternalCommand, so only update the exit status in output delegate if
             // it is nil. However, always update the status if the result is failed,