Skip to content

Allow differing leading/trailing whitespace when checking test output #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 16, 2016
Merged

Allow differing leading/trailing whitespace when checking test output #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0503412
Allow differing leading/trailing whitespace when checking test output
briancroom Mar 8, 2016
2b32c53
Take advantage of the new whitespace behavior in our test expectations
briancroom Mar 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Tests/Functional/ErrorHandling/main.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class ErrorHandling: XCTestCase {
}

// CHECK: Test Case 'ErrorHandling.test_shouldButDoesNotThrowErrorInAssertion' started.
// CHECK: .*/ErrorHandling/main.swift:\d+: error: ErrorHandling.test_shouldButDoesNotThrowErrorInAssertion : XCTAssertThrowsError failed: did not throw error -
// CHECK: .*/ErrorHandling/main.swift:\d+: error: ErrorHandling.test_shouldButDoesNotThrowErrorInAssertion : XCTAssertThrowsError failed: did not throw error -
// CHECK: Test Case 'ErrorHandling.test_shouldButDoesNotThrowErrorInAssertion' failed \(\d+\.\d+ seconds\).
func test_shouldButDoesNotThrowErrorInAssertion() {
XCTAssertThrowsError(try functionThatDoesNotThrowError())
Expand All @@ -60,7 +60,7 @@ class ErrorHandling: XCTestCase {
}

// CHECK: Test Case 'ErrorHandling.test_throwsErrorInAssertionButFailsWhenCheckingError' started.
// CHECK: .*/ErrorHandling/main.swift:\d+: error: ErrorHandling.test_throwsErrorInAssertionButFailsWhenCheckingError : XCTAssertEqual failed: \("Optional\("an error message"\)"\) is not equal to \("Optional\(""\)"\) -
// CHECK: .*/ErrorHandling/main.swift:\d+: error: ErrorHandling.test_throwsErrorInAssertionButFailsWhenCheckingError : XCTAssertEqual failed: \("Optional\("an error message"\)"\) is not equal to \("Optional\(""\)"\) -
// CHECK: Test Case 'ErrorHandling.test_throwsErrorInAssertionButFailsWhenCheckingError' failed \(\d+\.\d+ seconds\).
func test_throwsErrorInAssertionButFailsWhenCheckingError() {
XCTAssertThrowsError(try functionThatDoesThrowError()) { error in
Expand All @@ -77,7 +77,7 @@ class ErrorHandling: XCTestCase {
}

// CHECK: Test Case 'ErrorHandling.test_canAndDoesThrowErrorFromTestMethod' started.
// CHECK: \<EXPR\>:0: unexpected error: ErrorHandling.test_canAndDoesThrowErrorFromTestMethod : threw error "AnError\("an error message"\)" -
// CHECK: \<EXPR\>:0: unexpected error: ErrorHandling.test_canAndDoesThrowErrorFromTestMethod : threw error "AnError\("an error message"\)" -
// CHECK: Test Case 'ErrorHandling.test_canAndDoesThrowErrorFromTestMethod' failed \(\d+\.\d+ seconds\).
func test_canAndDoesThrowErrorFromTestMethod() throws {
try functionThatDoesThrowError()
Expand All @@ -94,7 +94,7 @@ class ErrorHandling: XCTestCase {
}

// CHECK: Test Case 'ErrorHandling.test_assertionExpressionCanThrow' started.
// CHECK: .*/ErrorHandling/main.swift:\d+: unexpected error: ErrorHandling.test_assertionExpressionCanThrow : XCTAssertEqual threw error "AnError\("did not actually return"\)" -
// CHECK: .*/ErrorHandling/main.swift:\d+: unexpected error: ErrorHandling.test_assertionExpressionCanThrow : XCTAssertEqual threw error "AnError\("did not actually return"\)" -
// CHECK: Test Case 'ErrorHandling.test_assertionExpressionCanThrow' failed \(\d+\.\d+ seconds\).
func test_assertionExpressionCanThrow() {
XCTAssertEqual(try functionThatShouldReturnButThrows(), 1)
Expand Down
12 changes: 6 additions & 6 deletions Tests/Functional/SelectedTest/main.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
// RUN: %{built_tests_dir}/SelectedTest SelectedTest.ExecutedTestCase/test_foo > %T/one_test_method || true
// RUN: %{built_tests_dir}/SelectedTest SelectedTest.ExecutedTestCase > %T/one_test_case || true
// RUN: %{built_tests_dir}/SelectedTest > %T/all || true
// RUN: %{xctest_checker} -p "// CHECK-METHOD: " %T/one_test_method %s
// RUN: %{xctest_checker} -p "// CHECK-TESTCASE: " %T/one_test_case %s
// RUN: %{xctest_checker} -p "// CHECK-ALL: " %T/all %s
// RUN: %{xctest_checker} -p "// CHECK-METHOD:" %T/one_test_method %s
// RUN: %{xctest_checker} -p "// CHECK-TESTCASE:" %T/one_test_case %s
// RUN: %{xctest_checker} -p "// CHECK-ALL:" %T/all %s

#if os(Linux) || os(FreeBSD)
import XCTest
Expand Down Expand Up @@ -44,11 +44,11 @@ class SkippedTestCase: XCTestCase {
return [("test_baz", test_baz)]
}

// CHECK-ALL: Test Case 'SkippedTestCase.test_baz' started.
// CHECK-ALL: Test Case 'SkippedTestCase.test_baz' passed \(\d+\.\d+ seconds\).
// CHECK-ALL: Test Case 'SkippedTestCase.test_baz' started.
// CHECK-ALL: Test Case 'SkippedTestCase.test_baz' passed \(\d+\.\d+ seconds\).
func test_baz() {}
}
// CHECK-ALL: Executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
// CHECK-ALL: Executed 1 test, with 0 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds

XCTMain([
testCase(ExecutedTestCase.allTests),
Expand Down
2 changes: 1 addition & 1 deletion Tests/Functional/SingleFailingTestCase/main.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class SingleFailingTestCase: XCTestCase {
}

// CHECK: Test Case 'SingleFailingTestCase.test_fails' started.
// CHECK: .*/SingleFailingTestCase/main.swift:22: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed -
// CHECK: .*/SingleFailingTestCase/main.swift:22: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed -
// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\).
func test_fails() {
XCTAssert(false)
Expand Down
10 changes: 10 additions & 0 deletions Tests/Functional/xctest_checker/tests/test_compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,15 @@ def test_includes_file_name_and_line_of_expected_in_error(self):

self.assertIn("{}:{}:".format(expected, 2), cm.exception.message)

def test_matching_ignores_leading_and_trailing_whitespace(self):
actual = _tmpfile('foo\nbar\nbaz\n')
expected = _tmpfile('c: foo\nc: bar \nc: baz\n')
compare.compare(actual, expected, check_prefix='c:')

def test_can_explicitly_match_leading_and_trailing_whitespace(self):
actual = _tmpfile('foo\n bar\nbaz \n')
expected = _tmpfile('c: foo\nc: ^ bar \nc: baz $\n')
compare.compare(actual, expected, check_prefix='c:')

if __name__ == "__main__":
unittest.main()
6 changes: 4 additions & 2 deletions Tests/Functional/xctest_checker/xctest_checker/compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@ def _expected_lines_and_line_numbers(path, check_prefix):
with open(path) as f:
for line_number, line in enumerate(f):
if line.startswith(check_prefix):
yield line[len(check_prefix):], line_number+1
yield line[len(check_prefix):].strip(), line_number+1

def _add_whitespace_leniency(original_regex):
return "^ *" + original_regex + " *$"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will the spaces here only match spaces, or will they match tabs as well? Perhaps use \s?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that's a good point. \s is probably more appropriate here considering the way that I've been referencing whitespace generally in the docs for this PR.

OTOH, considering that it's not actually necessary at this moment, and CI has gone green already with the existing code (on Linux at least), how about we make that adjustment separately?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Works for me! 👍


def compare(actual, expected, check_prefix):
"""
Expand All @@ -52,7 +54,7 @@ def compare(actual, expected, check_prefix):

(expected_line, expectation_source_line_number) = expected_line_and_line_number

if not re.match(expected_line, actual_line):
if not re.match(_add_whitespace_leniency(expected_line), actual_line):
raise AssertionError('Actual line did not match the expected '
'regular expression.\n'
'{}:{}: Actual: {}\n'
Expand Down
8 changes: 5 additions & 3 deletions Tests/Functional/xctest_checker/xctest_checker/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ def main():
parser.add_argument('expected', help='A path to a file containing the '
'expected output of an XCTest run.')
parser.add_argument('-p', '--check-prefix',
default='// CHECK: ',
default='// CHECK:',
help='{prog} checks actual output against expected '
'output. By default, {prog} only checks lines '
'that are prefixed with "// CHECK: ". This '
'that are prefixed with "// CHECK:". This '
'option can be used to change that '
'prefix.'.format(prog=parser.prog))
'prefix. Leading and trailing whitespace is '
'ignored unless the check line contains explicit '
'^ or $ characters'.format(prog=parser.prog))
args = parser.parse_args()
compare.compare(args.actual, args.expected, args.check_prefix)

Expand Down