Skip to content

Ensure xUnit/JUnit XML output correctly counts test-less issues. #597

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions Sources/Testing/Events/Recorder/Event.JUnitXMLRecorder.swift
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ extension Event {
/// This value does not include test suites.
var testCount = 0

/// Any recorded issues where the test was not known.
var issuesForUnknownTests = [Issue]()

/// A type describing data tracked on a per-test basis.
struct TestData: Sendable {
/// The ID of the test.
Expand Down Expand Up @@ -122,19 +125,22 @@ extension Event.JUnitXMLRecorder {
if issue.isKnown {
return nil
}
guard let id = test?.id else {
return nil // FIXME: handle issues without known tests
}
let keyPath = id.keyPathRepresentation
_context.withLock { context in
context.testData[keyPath]?.issues.append(issue)
if let id = test?.id {
let keyPath = id.keyPathRepresentation
_context.withLock { context in
context.testData[keyPath]?.issues.append(issue)
}
} else {
_context.withLock { context in
context.issuesForUnknownTests.append(issue)
}
}
return nil
case .runEnded:
return _context.withLock { context in
let issueCount = context.testData
.compactMap(\.value?.issues.count)
.reduce(into: 0, +=)
.reduce(into: 0, +=) + context.issuesForUnknownTests.count
let skipCount = context.testData
.compactMap(\.value?.skipInfo)
.count
Expand Down Expand Up @@ -218,7 +224,7 @@ extension Event.JUnitXMLRecorder {
">"
case "&":
"&"
case _ where !character.isASCII:
case _ where !character.isASCII || character.isNewline:
character.unicodeScalars.lazy
.map(\.value)
.map { "&#\($0);" }
Expand Down
28 changes: 26 additions & 2 deletions Tests/TestingTests/EventRecorderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -360,13 +360,32 @@ struct EventRecorderTests {
}
#endif

@Test("Recorded issues may not have associated tests")
func issueWithoutTest() {
@Test("HumanReadableOutputRecorder counts issues without associated tests")
func humanReadableRecorderCountsIssuesWithoutTests() {
let issue = Issue(kind: .unconditional, comments: [], sourceContext: .init())
let event = Event(.issueRecorded(issue), testID: nil, testCaseID: nil)
let context = Event.Context(test: nil, testCase: nil)

let recorder = Event.HumanReadableOutputRecorder()
let messages = recorder.record(event, in: context)
#expect(
messages.map(\.stringValue).contains { message in
message.contains("unknown")
}
)
}

@Test("JUnitXMLRecorder counts issues without associated tests")
func junitRecorderCountsIssuesWithoutTests() async throws {
let issue = Issue(kind: .unconditional, comments: [], sourceContext: .init())
let event = Event(.issueRecorded(issue), testID: nil, testCaseID: nil)
let context = Event.Context(test: nil, testCase: nil)

let recorder = Event.JUnitXMLRecorder { string in
if string.contains("<testsuite") {
#expect(string.contains(#"failures=1"#))
}
}
_ = recorder.record(event, in: context)
}
}
Expand All @@ -379,6 +398,7 @@ struct EventRecorderTests {
await { () async in
_ = Issue.record("Whales fail asynchronously.")
}()
Issue.record("Whales\nalso\nfall.")
}
@Test(.hidden) func expectantKangaroo() {
#expect("abc" == "xyz")
Expand Down Expand Up @@ -449,6 +469,10 @@ struct EventRecorderTests {
@Test(.hidden) func cornyUnicorn🦄() throws {
throw MyDescriptiveError(description: #"🦄"#)
}

@Test(.hidden) func burdgeoningBudgerigar() {
Issue.record(#"</>& "Down\#nwe\#ngo!""#)
}
}

@Suite(.hidden) struct PredictablyFailingTests {
Expand Down