Enable integration tests in self-hosted CI jobs (#3088)

These were initially added in #2592 but not enabled because of lack of necessary Xcode support from Swift CI.

- Add ability to conditionally skip when self-hosted
- Skip failing tests and add more details about reasons

Author: friedbunny

IntegrationTests/Fixtures/XCBuild/ExecutableProducts/Bar/Sources/cbar/main.c

@@ -1,3 +1,5 @@
+#include <stdio.h>
+
 int main(void) {
     printf("Hello from cbar");
-}
+}

IntegrationTests/Fixtures/XCBuild/ExecutableProducts/Foo/Sources/cfoo/main.c

@@ -1,3 +1,5 @@
+#include <stdio.h>
+
 int main(void) {
     printf("Hello from cfoo");
-}
+}

IntegrationTests/Package.swift

@@ -16,7 +16,7 @@ import class Foundation.ProcessInfo
 
 if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
     package.dependencies += [
-        .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("master")),
+        .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("main")),
     ]
 } else {
     package.dependencies += [

IntegrationTests/Tests/IntegrationTests/BasicTests.swift

@@ -17,10 +17,9 @@ final class BasicTests: XCTestCase {
         XCTAssertMatch(try sh(swift, "--version").stdout, .contains("Swift version"))
     }
 
-    // Disabled because these packages don't use the latest runtime library which doesn't work with self-hosted builds.
-    //
-    // FIXME: We should to use XCTSkip to skip this test instead.
-    func DISABLED_testExamplePackageDealer() throws {
+    func testExamplePackageDealer() throws {
+        try XCTSkipIf(isSelfHosted, "These packages don't use the latest runtime library, which doesn't work with self-hosted builds.")
+
         try withTemporaryDirectory { dir in
             let dealerDir = dir.appending(component: "dealer")
             try sh("git", "clone", "https://github.com/apple/example-package-dealer", dealerDir)
@@ -120,8 +119,9 @@ final class BasicTests: XCTestCase {
         }
     }
 
-    // TODO: Check why this test is failing to test in Xcode and through the command line.
-    func _testSwiftPackageInitLib() throws {
+    func testSwiftPackageInitLib() throws {
+        try XCTSkip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI")
+
         try withTemporaryDirectory { dir in
             // Create a new package with an executable target.
             let projectDir = dir.appending(component: "Project")
@@ -203,6 +203,8 @@ final class BasicTests: XCTestCase {
     }
 
     func testSwiftTest() throws {
+        try XCTSkip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI")
+
         try withTemporaryDirectory { dir in
             let toolDir = dir.appending(component: "swiftTest")
             try localFileSystem.createDirectory(toolDir)
@@ -234,6 +236,8 @@ final class BasicTests: XCTestCase {
     }
 
     func testSwiftTestWithResources() throws {
+        try XCTSkip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI")
+
         try withTemporaryDirectory { dir in
             let toolDir = dir.appending(component: "swiftTestResources")
             try localFileSystem.createDirectory(toolDir)

IntegrationTests/Tests/IntegrationTests/Helpers.swift

@@ -35,11 +35,11 @@ let toolchainPath: AbsolutePath = {
 
   #if os(macOS)
     let swiftcPath = try! AbsolutePath(sh("xcrun", "--find", "swift").stdout.spm_chomp())
-    let toolchainPath = swiftcPath.parentDirectory.parentDirectory.parentDirectory
-    return toolchainPath
   #else
-    fatalError("TOOLCHAIN_PATH environment variable required")
+    let swiftcPath = try! AbsolutePath(sh("which", "swift").stdout.spm_chomp())
   #endif
+    let toolchainPath = swiftcPath.parentDirectory.parentDirectory.parentDirectory
+    return toolchainPath
 }()
 
 let clang: AbsolutePath = {
@@ -89,7 +89,7 @@ let lldb: AbsolutePath = {
         return toolchainLLDBPath
     }
 
-    #if os(macOS)
+  #if os(macOS)
     let lldbPath = try! AbsolutePath(sh("xcrun", "--find", "lldb").stdout.spm_chomp())
     return lldbPath
   #else
@@ -121,6 +121,10 @@ let swiftRun: AbsolutePath = {
     return swiftpmBinaryDirectory.appending(component: "swift-run")
 }()
 
+let isSelfHosted: Bool = {
+    return ProcessInfo.processInfo.environment["SWIFTCI_IS_SELF_HOSTED"] != nil
+}()
+
 @discardableResult
 func sh(
     _ arguments: CustomStringConvertible...,
@@ -313,6 +317,6 @@ func binaryTargetsFixture(_ closure: (AbsolutePath) throws -> Void) throws {
     }
 }
 
-func XCTSkip() throws {
-    throw XCTSkip()
+func XCTSkip(_ message: String? = nil) throws {
+    throw XCTSkip(message)
 }

IntegrationTests/Tests/IntegrationTests/SwiftPMTests.swift

@@ -13,9 +13,13 @@ import TSCBasic
 import TSCTestSupport
 
 final class SwiftPMTests: XCTestCase {
-  #if os(macOS)
-    // FIXME: This is failing right now.
-    func DISABLED_testBinaryTargets() throws {
+    func testBinaryTargets() throws {
+        try XCTSkip("FIXME: ld: warning: dylib (/../BinaryTargets.6YVYK4/TestBinary/.build/x86_64-apple-macosx/debug/SwiftFramework.framework/SwiftFramework) was built for newer macOS version (10.15) than being linked (10.10)")
+
+        #if !os(macOS)
+            try XCTSkip("Test requires macOS")
+        #endif
+
         try binaryTargetsFixture { prefix in
             do {
                 let (stdout, stderr) = try sh(swiftRun, "--package-path", prefix, "exe")
@@ -46,6 +50,10 @@ final class SwiftPMTests: XCTestCase {
     }
 
     func testArchCustomization() throws {
+        #if !os(macOS)
+            try XCTSkip("Test requires macOS")
+        #endif
+
         try withTemporaryDirectory { tmpDir in
             let foo = tmpDir.appending(component: "foo")
             try localFileSystem.createDirectory(foo)
@@ -55,7 +63,7 @@ final class SwiftPMTests: XCTestCase {
             try localFileSystem.writeFileContents(foo.appending(RelativePath("Sources/foo/main.m"))) {
                 $0 <<< "int main() {}"
             }
-            let archs = ["x86_64", "x86_64h"]
+            let archs = ["x86_64", "arm64"]
 
             for arch in archs {
                 try sh(swiftBuild, "--package-path", foo, "--arch", arch)
@@ -75,5 +83,4 @@ final class SwiftPMTests: XCTestCase {
             }
         }
     }
-  #endif
 }

IntegrationTests/Tests/IntegrationTests/XCBuildTests.swift

@@ -16,7 +16,7 @@ import TSCTestSupport
 final class XCBuildTests: XCTestCase {
     func testExecutableProducts() throws {
         #if !os(macOS)
-              try XCTSkip()
+            try XCTSkip("Test requires macOS")
         #endif
 
         fixture(name: "XCBuild/ExecutableProducts") { path in
@@ -117,7 +117,7 @@ final class XCBuildTests: XCTestCase {
 
     func testTestProducts() throws {
         #if !os(macOS)
-            try XCTSkip()
+            try XCTSkip("Test requires macOS")
         #endif
 
         fixture(name: "XCBuild/TestProducts") { path in
@@ -199,7 +199,7 @@ final class XCBuildTests: XCTestCase {
 
     func testLibraryProductsAndTargets() throws {
         #if !os(macOS)
-            try XCTSkip()
+            try XCTSkip("Test requires macOS")
         #endif
 
         fixture(name: "XCBuild/Libraries") { path in
@@ -272,8 +272,10 @@ final class XCBuildTests: XCTestCase {
     }
 
     func testSystemTargets() throws {
+        try XCTSkip("FIXME: ld: warning: ignoring file /../XCBuild_SystemTargets.b38QoO/Inputs/libsys.a, building for macOS-arm64 but attempting to link with file built for unknown-x86_64\n\nUndefined symbols for architecture arm64:\n  \"_GetSystemLibName\", referenced from:\n      _main in main.o\n\nld: symbol(s) not found for architecture arm64\n\nclang: error: linker command failed with exit code 1 (use -v to see invocation)\n\nBuild cancelled\n")
+
         #if !os(macOS)
-            try XCTSkip()
+            try XCTSkip("Test requires macOS")
         #endif
 
         fixture(name: "XCBuild/SystemTargets") { path in
@@ -298,17 +300,18 @@ final class XCBuildTests: XCTestCase {
     }
 
     func testBinaryTargets() throws {
-        //FIXME: This test randomly succeeds or fails, depending on the order the subtasks are executed in.
-        try XCTSkip()
+        try XCTSkip("FIXME: This test randomly succeeds or fails, depending on the order the subtasks are executed in.")
 
         try binaryTargetsFixture { path in
             try sh(swiftBuild, "--package-path", path, "-c", "debug", "--build-system", "xcode", "--target", "exe")
         }
     }
 
     func testSwiftTest() throws {
+        try XCTSkip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI")
+
         #if !os(macOS) || Xcode
-            try XCTSkip()
+            try XCTSkip("Test requires macOS")
         #endif
 
         fixture(name: "XCBuild/TestProducts") { path in

Utilities/build-using-self

@@ -8,6 +8,7 @@ cd "${__dir}/.."
 echo "Current directory is ${PWD}"
 
 CONFIGURATION=debug
+export SWIFTCI_IS_SELF_HOSTED=1
 
 set -x
 
@@ -16,9 +17,7 @@ swift package update
 swift build -c $CONFIGURATION
 swift test -c $CONFIGURATION --parallel --enable-test-discovery
 
-# FIXME: Disabled until we get some version of Xcode 11.4 on Swift CI.
-#
 # Run the integration tests with just built SwiftPM.
-# export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path)
-# cd IntegrationTests
-# $SWIFTPM_BIN_DIR/swift-test --parallel --enable-test-discovery
+export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path)
+cd IntegrationTests
+$SWIFTPM_BIN_DIR/swift-test --parallel --enable-test-discovery