Skip to content

Commit 4f1349c

Browse files
bkhouribkhouri
committed
Update xUnit to display output on failures (XCTest only)
For XCTest, the generated xUnit XML file is not helpful when tests fail as it contains the message "failure", which is redundant with the test results. Update the XML output to dipslay the result output instead of static "failure" message.
1 parent dca0cc2 commit 4f1349c

File tree

19 files changed

+441
-9
lines changed

19 files changed

+441
-9
lines changed

‎Fixtures/Miscellaneous/TestMultipleFailureSwiftTesting/.gitignore

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
.DS_Store
2+
/.build
3+
/Packages
4+
xcuserdata/
5+
DerivedData/
6+
.swiftpm/configuration/registries.json
7+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
8+
.netrc

‎Fixtures/Miscellaneous/TestMultipleFailureSwiftTesting/Package.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// swift-tools-version: 6.0
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "TestMultipleFailureSwiftTesting",
8+
products: [
9+
// Products define the executables and libraries a package produces, making them visible to other packages.
10+
.library(
11+
name: "TestMultipleFailureSwiftTesting",
12+
targets: ["TestMultipleFailureSwiftTesting"]),
13+
],
14+
targets: [
15+
// Targets are the basic building blocks of a package, defining a module or a test suite.
16+
// Targets can depend on other targets in this package and products from dependencies.
17+
.target(
18+
name: "TestMultipleFailureSwiftTesting"),
19+
.testTarget(
20+
name: "TestMultipleFailureSwiftTestingTests",
21+
dependencies: ["TestMultipleFailureSwiftTesting"]
22+
),
23+
]
24+
)

‎Fixtures/Miscellaneous/TestMultipleFailureSwiftTesting/Sources/TestMultipleFailureSwiftTesting/TestMultipleFailureSwiftTesting.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// The Swift Programming Language
2+
// https://docs.swift.org/swift-book

‎Fixtures/Miscellaneous/TestMultipleFailureSwiftTesting/Tests/TestMultipleFailureSwiftTestingTests/TestMultipleFailureSwiftTestingTests.swift

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import Testing
2+
@testable import TestMultipleFailureSwiftTesting
3+
4+
@Test func testFailure1() async throws {
5+
#expect(Bool(false), "ST Test failure 1")
6+
}
7+
8+
@Test func testFailure2() async throws {
9+
#expect(Bool(false), "ST Test failure 2")
10+
}
11+
12+
@Test func testFailure3() async throws {
13+
#expect(Bool(false), "ST Test failure 3")
14+
}
15+
16+
@Test func testFailure4() async throws {
17+
#expect(Bool(false), "ST Test failure 4")
18+
}
19+
20+
@Test func testFailure5() async throws {
21+
#expect(Bool(false), "ST Test failure 5")
22+
}
23+
24+
@Test func testFailure6() async throws {
25+
#expect(Bool(false), "ST Test failure 6")
26+
}
27+
28+
@Test func testFailure7() async throws {
29+
#expect(Bool(false), "ST Test failure 7")
30+
}
31+
32+
@Test func testFailure8() async throws {
33+
#expect(Bool(false), "ST Test failure 8")
34+
}
35+
36+
@Test func testFailure9() async throws {
37+
#expect(Bool(false), "ST Test failure 9")
38+
}
39+
40+
@Test func testFailure10() async throws {
41+
#expect(Bool(false), "ST Test failure 10")
42+
}

‎Fixtures/Miscellaneous/TestMultipleFailureXCTest/.gitignore

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
.DS_Store
2+
/.build
3+
/Packages
4+
xcuserdata/
5+
DerivedData/
6+
.swiftpm/configuration/registries.json
7+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
8+
.netrc

‎Fixtures/Miscellaneous/TestMultipleFailureXCTest/Package.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// swift-tools-version: 6.0
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "TestMultipleFailureXCTest",
8+
products: [
9+
// Products define the executables and libraries a package produces, making them visible to other packages.
10+
.library(
11+
name: "TestMultipleFailureXCTest",
12+
targets: ["TestMultipleFailureXCTest"]),
13+
],
14+
targets: [
15+
// Targets are the basic building blocks of a package, defining a module or a test suite.
16+
// Targets can depend on other targets in this package and products from dependencies.
17+
.target(
18+
name: "TestMultipleFailureXCTest"),
19+
.testTarget(
20+
name: "TestMultipleFailureXCTestTests",
21+
dependencies: ["TestMultipleFailureXCTest"]
22+
),
23+
]
24+
)

‎Fixtures/Miscellaneous/TestMultipleFailureXCTest/Sources/TestMultipleFailureXCTest/TestMultipleFailureXCTest.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// The Swift Programming Language
2+
// https://docs.swift.org/swift-book

‎Fixtures/Miscellaneous/TestMultipleFailureXCTest/Tests/TestMultipleFailureXCTestTests/TestMultipleFailureXCTestTests.swift

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import XCTest
2+
@testable import TestMultipleFailureXCTest
3+
4+
final class TestMultipleFailureXCTestTests: XCTestCase {
5+
func testFailure1() throws {
6+
XCTAssertFalse(true, "Test failure 1")
7+
}
8+
9+
func testFailure2() throws {
10+
XCTAssertFalse(true, "Test failure 2")
11+
}
12+
13+
func testFailure3() throws {
14+
XCTAssertFalse(true, "Test failure 3")
15+
}
16+
17+
func testFailure4() throws {
18+
XCTAssertFalse(true, "Test failure 4")
19+
}
20+
21+
func testFailure5() throws {
22+
XCTAssertFalse(true, "Test failure 5")
23+
}
24+
25+
func testFailure6() throws {
26+
XCTAssertFalse(true, "Test failure 6")
27+
}
28+
29+
func testFailure7() throws {
30+
XCTAssertFalse(true, "Test failure 7")
31+
}
32+
33+
func testFailure8() throws {
34+
XCTAssertFalse(true, "Test failure 8")
35+
}
36+
37+
func testFailure9() throws {
38+
XCTAssertFalse(true, "Test failure 9")
39+
}
40+
41+
func testFailure10() throws {
42+
XCTAssertFalse(true, "Test failure 10")
43+
}
44+
45+
}

‎Fixtures/Miscellaneous/TestSingleFailureSwiftTesting/.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.DS_Store
2+
/.build
3+
/.index-build
4+
/Packages
5+
xcuserdata/
6+
DerivedData/
7+
.swiftpm/configuration/registries.json
8+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
9+
.netrc

‎Fixtures/Miscellaneous/TestSingleFailureSwiftTesting/Package.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// swift-tools-version: 5.9
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "TestFailuresSwiftTesting",
8+
products: [
9+
// Products define the executables and libraries a package produces, making them visible to other packages.
10+
.library(
11+
name: "TestFailuresSwiftTesting",
12+
targets: ["TestFailuresSwiftTesting"])
13+
],
14+
targets: [
15+
// Targets are the basic building blocks of a package, defining a module or a test suite.
16+
// Targets can depend on other targets in this package and products from dependencies.
17+
.target(
18+
name: "TestFailuresSwiftTesting"),
19+
.testTarget(
20+
name: "TestFailuresSwiftTestingTests",
21+
dependencies: ["TestFailuresSwiftTesting"]
22+
)
23+
]
24+
)

‎Fixtures/Miscellaneous/TestSingleFailureSwiftTesting/Sources/TestFailuresSwiftTesting/TestFailuresSwiftTesting.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// The Swift Programming Language
2+
// https://docs.swift.org/swift-book

‎Fixtures/Miscellaneous/TestSingleFailureSwiftTesting/Tests/TestFailuresSwiftTestingTests/TestFailuresSwiftTestingTests.swift

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import Testing
2+
@testable import TestFailuresSwiftTesting
3+
4+
@Test func example() async throws {
5+
#expect(Bool(false), "Purposely failing & validating XML espace \"'<>")
6+
}

‎Fixtures/Miscellaneous/TestSingleFailureXCTest/.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.DS_Store
2+
/.build
3+
/.index-build
4+
/Packages
5+
xcuserdata/
6+
DerivedData/
7+
.swiftpm/configuration/registries.json
8+
.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
9+
.netrc

‎Fixtures/Miscellaneous/TestSingleFailureXCTest/Package.swift

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// swift-tools-version: 5.9
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "TestFailures",
8+
products: [
9+
// Products define the executables and libraries a package produces, making them visible to other packages.
10+
.library(
11+
name: "TestFailures",
12+
targets: ["TestFailures"])
13+
],
14+
targets: [
15+
// Targets are the basic building blocks of a package, defining a module or a test suite.
16+
// Targets can depend on other targets in this package and products from dependencies.
17+
.target(
18+
name: "TestFailures"),
19+
.testTarget(
20+
name: "TestFailuresTests",
21+
dependencies: ["TestFailures"]
22+
)
23+
]
24+
)

‎Fixtures/Miscellaneous/TestSingleFailureXCTest/Sources/TestFailures/TestFailures.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// The Swift Programming Language
2+
// https://docs.swift.org/swift-book

‎Fixtures/Miscellaneous/TestSingleFailureXCTest/Tests/TestFailuresTests/TestFailuresTests.swift

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import XCTest
2+
@testable import TestFailures
3+
4+
final class TestFailuresTests: XCTestCase {
5+
func testExample() throws {
6+
XCTAssertFalse(true, "Purposely failing & validating XML espace \"'<>")
7+
}
8+
}
9+

‎Sources/Commands/SwiftTestCommand.swift

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,12 @@ struct TestCommandOptions: ParsableArguments {
183183
help: "Path where the xUnit xml file should be generated.")
184184
var xUnitOutput: AbsolutePath?
185185

186+
@Flag(
187+
name: .customLong("experimental-xunit-message-failure"),
188+
help: "When Set, enabled an experimental message failure content (XCTest only)."
189+
)
190+
var shouldShowDetailedFailureMessage: Bool = false
191+
186192
/// Generate LinuxMain entries and exit.
187193
@Flag(name: .customLong("testable-imports"), inversion: .prefixedEnableDisable, help: "Enable or disable testable imports. Enabled by default.")
188194
var enableTestableImports: Bool = true
@@ -325,7 +331,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
325331
result = runner.ranSuccessfully ? .success : .failure
326332
}
327333

328-
try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState)
334+
try generateXUnitOutputIfRequested(
335+
for: testResults,
336+
swiftCommandState: swiftCommandState,
337+
detailedFailureMessage: self.options.shouldShowDetailedFailureMessage
338+
)
329339
results.append(result)
330340
}
331341
}
@@ -401,7 +411,8 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
401411
/// Generate xUnit file if requested.
402412
private func generateXUnitOutputIfRequested(
403413
for testResults: [ParallelTestRunner.TestResult],
404-
swiftCommandState: SwiftCommandState
414+
swiftCommandState: SwiftCommandState,
415+
detailedFailureMessage: Bool
405416
) throws {
406417
guard let xUnitOutput = options.xUnitOutput else {
407418
return
@@ -411,7 +422,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
411422
fileSystem: swiftCommandState.fileSystem,
412423
results: testResults
413424
)
414-
try generator.generate(at: xUnitOutput)
425+
try generator.generate(at: xUnitOutput, detailedFailureMessage: detailedFailureMessage)
415426
}
416427

417428
// MARK: - Common implementation
@@ -1366,7 +1377,7 @@ final class XUnitGenerator {
13661377
}
13671378

13681379
/// Generate the file at the given path.
1369-
func generate(at path: AbsolutePath) throws {
1380+
func generate(at path: AbsolutePath, detailedFailureMessage: Bool) throws {
13701381
var content =
13711382
"""
13721383
<?xml version="1.0" encoding="UTF-8"?>
@@ -1399,7 +1410,15 @@ final class XUnitGenerator {
13991410
"""
14001411

14011412
if !result.success {
1402-
content += "<failure message=\"failed\"></failure>\n"
1413+
var failureMessage: String = "failed"
1414+
if detailedFailureMessage {
1415+
failureMessage = result.output
1416+
failureMessage.replace("&", with: "&amp;")
1417+
failureMessage.replace("\"", with:"&quot;")
1418+
failureMessage.replace(">", with: "&gt;")
1419+
failureMessage.replace("<", with: "&lt;")
1420+
}
1421+
content += "<failure message=\"\(failureMessage)\"></failure>\n"
14031422
}
14041423

14051424
content += "</testcase>\n"

‎Sources/_InternalTestSupport/SwiftPMProduct.swift

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ extension SwiftPM {
8282
public func execute(
8383
_ args: [String] = [],
8484
packagePath: AbsolutePath? = nil,
85-
env: Environment? = nil
85+
env: Environment? = nil,
86+
errorIfCommandUnsuccessful: Bool = true
8687
) async throws -> (stdout: String, stderr: String) {
8788
let result = try await executeProcess(
8889
args,
@@ -93,8 +94,11 @@ extension SwiftPM {
9394
let stdout = try result.utf8Output()
9495
let stderr = try result.utf8stderrOutput()
9596

97+
let returnValue = (stdout: stdout, stderr: stderr)
98+
if (!errorIfCommandUnsuccessful) { return returnValue }
99+
96100
if result.exitStatus == .terminated(code: 0) {
97-
return (stdout: stdout, stderr: stderr)
101+
return returnValue
98102
}
99103
throw SwiftPMError.executionFailure(
100104
underlying: AsyncProcessResult.Error.nonZeroExit(result),

0 commit comments

Comments
 (0)