[Tests] Add functional tests

README.md

@@ -38,6 +38,23 @@ If your install of Swift is located at `/swift` and you wish to install XCTest i
 ./build_script.py --swiftc="/swift/usr/bin/swiftc" --build-dir="/tmp/XCTest_build" --swift-build-dir="/swift/usr" --library-install-path="/swift/usr/lib/swift/linux" --module-install-path="/swift/usr/lib/swift/linux/x86_64"
 ```
 
+To run the tests on Linux, pass the `--test` option in combination with options to
+install XCTest in your active version of Swift:
+
+```sh
+./build_script.py \
+    --swiftc="/swift/usr/bin/swiftc" \
+    --build-dir="/tmp/XCTest_build" \
+    --swift-build-dir="/swift/usr" \
+    --library-install-path="/swift/usr/lib/swift/linux" \
+    --module-install-path="/swift/usr/lib/swift/linux/x86_64" \
+    --test
+```
+
+To run the tests on OS X, build and run the `SwiftXCTestFunctionalTests` target in the Xcode project.
+
+You may add tests for XCTest by including them in the `Tests/Functional/` directory. For an example, see `Tests/Functional/SingleFailingTestCase`.
+
 ### Additional Considerations for Swift on Linux
 
 When running on the Objective-C runtime, XCTest is able to find all of your tests by simply asking the runtime for the subclasses of `XCTestCase`. It then finds the methods that start with the string `test`. This functionality is not currently present when running on the Swift runtime. Therefore, you must currently provide an additional property called `allTests` in your `XCTestCase` subclass. This method lists all of the tests in the test class. The rest of your test case subclass still contains your test methods.

Tests/Functional/SingleFailingTestCase/main.swift

@@ -0,0 +1,28 @@
+// RUN: %{swiftc} %s -o %{built_tests_dir}/SingleFailingTestCase
+// RUN: %{built_tests_dir}/SingleFailingTestCase > %{test_output} || true
+// RUN: %{xctest_checker} %{test_output} %s
+// CHECK: Test Case 'SingleFailingTestCase.test_fails' started.
+// CHECK: .*/Tests/Functional/SingleFailingTestCase/main.swift:24: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed - 
+// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\).
+// CHECK: Executed 1 test, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Total executed 1 test, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+
+#if os(Linux) || os(FreeBSD)
+    import XCTest
+#else
+    import SwiftXCTest
+#endif
+
+class SingleFailingTestCase: XCTestCase {
+    var allTests: [(String, () -> ())] {
+        return [
+            ("test_fails", test_fails),
+        ]
+    }
+
+    func test_fails() {
+        XCTAssert(false)
+    }
+}
+
+XCTMain([SingleFailingTestCase()])

Tests/Functional/lit.cfg

@@ -0,0 +1,73 @@
+import os
+import platform
+import tempfile
+
+import lit
+
+# Set up lit config.
+config.name = 'SwiftXCTestFunctionalTests'
+config.test_format = lit.formats.ShTest(execute_external=False)
+config.suffixes = ['.swift']
+
+# Set up the substitutions used by the functional test suite.
+
+# First, our tests need a way to compile source files into
+# executables that are linked against swift-corelibs-xctest.
+# We'll provide one via the %swiftc substitution.
+#
+# Linux tests are run after swift-corelibs-xctest is installed
+# in the Swift library path, so we only need the path to `swiftc`
+# in order to compile.
+def _getenv(name):
+    value = os.getenv(name, None)
+    if value is None:
+        lit_config.fatal(
+            'Environment variable ${} is not set.\n'
+            '$SWIFT_EXEC, $SDKROOT, $BUILT_PRODUCTS_DIR, $PLATFORM_NAME, and '
+            '$MACOSX_DEPLOYMENT_TARGET must all be set for lit tests to '
+            'run.'.format(name))
+    return value
+
+swift_exec = [_getenv('SWIFT_EXEC')]
+
+if platform.system() == 'Darwin':
+    # On OS X, we need to make sure swiftc references the
+    # proper SDK, has a deployment target set, and more...
+    # Here we rely on environment variables, produced by xcodebuild.
+    sdk_root = _getenv('SDKROOT')
+    built_products_dir = _getenv('BUILT_PRODUCTS_DIR')
+    platform_name = _getenv('PLATFORM_NAME')
+    deployment_target = _getenv('MACOSX_DEPLOYMENT_TARGET')
+
+    target = '{}-apple-{}{}'.format(
+        platform.machine(), platform_name, deployment_target)
+    swift_exec.extend([
+        '-sdk', sdk_root,
+        '-target', target,
+        '-L', built_products_dir,
+        '-I', built_products_dir,
+        '-F', built_products_dir,
+        '-Xlinker', '-rpath',
+        '-Xlinker', built_products_dir])
+
+# Having prepared the swiftc command, we set the substitution.
+config.substitutions.append(('%{swiftc}', ' '.join(swift_exec)))
+
+# Add the %built_tests_dir substitution, which is a temporary
+# directory used to store built files.
+built_tests_dir = tempfile.mkdtemp()
+config.substitutions.append(('%{built_tests_dir}', built_tests_dir))
+
+# Add the %test_output substitution, which is a temporary file
+# used to store test output.
+test_output = tempfile.mkstemp()[1]
+config.substitutions.append(('%{test_output}', test_output))
+
+# Add the %xctest_checker substitution, which is a Python script
+# can be used to compare the actual XCTest output to the expected
+# output.
+xctest_checker = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'xctest_checker',
+    'xctest_checker.py')
+config.substitutions.append(('%{xctest_checker}', xctest_checker))

Tests/Functional/xctest_checker/.gitignore

@@ -0,0 +1,26 @@
+# Compile artifacts
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution/packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt

Tests/Functional/xctest_checker/setup.py

@@ -0,0 +1,43 @@
+import os
+import setuptools
+
+import xctest_checker
+
+# setuptools expects to be invoked from within the directory of setup.py,
+# but it is nice to allow `python path/to/setup.py install` to work
+# (for scripts, etc.)
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+setuptools.setup(
+    name='xctest_checker',
+    version=xctest_checker.__version__,
+
+    author=xctest_checker.__author__,
+    author_email=xctest_checker.__email__,
+    url='http://swift.org',
+    license='Apache',
+
+    description="A tool to verify the output of XCTest runs.",
+    keywords='test xctest swift',
+
+    test_suite='tests',
+
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Testing',
+    ],
+
+    zip_safe=False,
+    packages=setuptools.find_packages(),
+    entry_points={
+        'console_scripts': [
+            'xctest_checker = xctest_checker:main',
+        ],
+    }
+)

Tests/Functional/xctest_checker/tests/test_compare.py

@@ -0,0 +1,41 @@
+import tempfile
+import unittest
+
+from xctest_checker import compare
+
+
+def _tmpfile(content):
+    """Returns the path to a temp file with the given contents."""
+    tmp = tempfile.mkstemp()[1]
+    with open(tmp, 'w') as f:
+        f.write(content)
+    return tmp
+
+
+class CompareTestCase(unittest.TestCase):
+    def test_no_match_raises(self):
+        actual = _tmpfile('foo\nbar\nbaz\n')
+        expected = _tmpfile('c: foo\nc: baz\nc: bar\n')
+        with self.assertRaises(AssertionError):
+            compare.compare(actual, expected, check_prefix='c: ')
+
+    def test_too_few_expected_raises(self):
+        actual = _tmpfile('foo\nbar\nbaz\n')
+        expected = _tmpfile('c: foo\nc: bar\n')
+        with self.assertRaises(AssertionError):
+            compare.compare(actual, expected, check_prefix='c: ')
+
+    def test_too_many_expected_raises(self):
+        actual = _tmpfile('foo\nbar\n')
+        expected = _tmpfile('c: foo\nc: bar\nc: baz\n')
+        with self.assertRaises(AssertionError):
+            compare.compare(actual, expected, check_prefix='c: ')
+
+    def test_match_does_not_raise(self):
+        actual = _tmpfile('foo\nbar\nbaz\n')
+        expected = _tmpfile('c: foo\nc: bar\nc: baz\n')
+        compare.compare(actual, expected, check_prefix='c: ')
+
+
+if __name__ == "__main__":
+    unittest.main()

Tests/Functional/xctest_checker/xctest_checker.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import xctest_checker
+
+if __name__ == '__main__':
+    xctest_checker.main()

Tests/Functional/xctest_checker/xctest_checker/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import absolute_import
+from .main import main
+
+__author__ = 'Brian Gesiak'
+__email__ = '[email protected]'
+__versioninfo__ = (0, 1, 0)
+__version__ = '.'.join(str(v) for v in __versioninfo__)
+
+__all__ = []

Tests/Functional/xctest_checker/xctest_checker/compare.py

@@ -0,0 +1,46 @@
+import re
+
+
+def _actual_lines(path):
+    """
+    Returns a generator that yields each line in the file at the given path.
+    """
+    with open(path) as f:
+        for line in f:
+            yield line
+
+
+def _expected_lines(path, check_prefix):
+    """
+    Returns a generator that yields each line in the file at the given path
+    that begins with the given prefix.
+    """
+    with open(path) as f:
+        for line in f:
+            if line.startswith(check_prefix):
+                yield line[len(check_prefix):]
+
+
+def compare(actual, expected, check_prefix):
+    """
+    Compares each line in the two given files.
+    If any line in the 'actual' file doesn't match the regex in the 'expected'
+    file, raises an AssertionError. Also raises an AssertionError if the number
+    of lines in the two files differ.
+    """
+    for actual_line, expected_line in map(
+            None,
+            _actual_lines(actual),
+            _expected_lines(expected, check_prefix)):
+        if actual_line is None:
+            raise AssertionError('There were more lines expected to appear '
+                                 'than there were lines in the actual input.')
+        if expected_line is None:
+            raise AssertionError('There were more lines than expected to '
+                                 'appear.')
+        if not re.match(expected_line, actual_line):
+            raise AssertionError('Actual line did not match the expected '
+                                 'regular expression.\n'
+                                 'Actual: {}\n'
+                                 'Expected: {}\n'.format(
+                                     repr(actual_line), repr(expected_line)))

Tests/Functional/xctest_checker/xctest_checker/main.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+
+import argparse
+
+from . import compare
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('actual', help='A path to a file containing the '
+                                       'actual output of an XCTest run.')
+    parser.add_argument('expected', help='A path to a file containing the '
+                                         'expected output of an XCTest run.')
+    parser.add_argument('-p', '--check-prefix',
+                        default='// CHECK: ',
+                        help='{prog} checks actual output against expected '
+                             'output. By default, {prog} only checks lines '
+                             'that are prefixed with "// CHECK: ". This '
+                             'option can be used to change that '
+                             'prefix.'.format(prog=parser.prog))
+    args = parser.parse_args()
+    compare.compare(args.actual, args.expected, args.check_prefix)
+
+
+if __name__ == '__main__':
+    main()