[Tests] Add functional tests

Adds a test runner (lit) that compares annotations in the source code of an
XCTest file to actual output when that source code is compiled and run.
This acts as a regression test suite for the project.

- Because FileCheck is not available unless LLVM is built, includes an
  XCTest-specific version of FileCheck called `xctest_checker`. This is
  used by the lit tests in order to verify the output of XCTest runs.
- Adds a test case that verifies the output of a single failing test
  case.
- To run the tests on Linux, adds a `--test` option to
  `build_script.py`.
- To run the tests on OS X, adds a `SwiftXCTestFunctionalTests` target
  to the Xcode project.

Author: modocache

Tests/Functional/SingleFailingTestCase/main.swift

@@ -0,0 +1,28 @@
+// RUN: %swift_exec %s -o %built_tests_dir/SingleFailingTestCase
+// RUN: %built_tests_dir/SingleFailingTestCase > %test_output || true
+// RUN: %xctest-checker %test_output %s
+// CHECK: Test Case 'SingleFailingTestCase.test_fails' started.
+// CHECK: .*/Tests/Functional/SingleFailingTestCase/main.swift:24: error: SingleFailingTestCase.test_fails : 
+// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\).
+// CHECK: Executed 1 test, with 1 failure 0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+// CHECK: Total executed 1 test, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
+
+#if os(Linux)
+    import XCTest
+#else
+    import SwiftXCTest
+#endif
+
+class SingleFailingTestCase: XCTestCase {
+    var allTests: [(String, () -> ())] {
+        return [
+            ("test_fails", test_fails),
+        ]
+    }
+
+    func test_fails() {
+        XCTAssert(false)
+    }
+}
+
+XCTMain([SingleFailingTestCase()])

Tests/Functional/lit.cfg

@@ -0,0 +1,73 @@
+import os
+import platform
+import tempfile
+
+import lit
+
+# Set up lit config.
+config.name = 'SwiftXCTestFunctionalTests'
+config.test_format = lit.formats.ShTest(execute_external=False)
+config.suffixes = ['.swift']
+
+# Set up the substitutions used by the functional test suite.
+
+# First, our tests need a way to compile source files into
+# executables that are linked against swift-corelibs-xctest.
+#
+# Linux tests are run after swift-corelibs-xctest is installed
+# in the Swift library path, so we only need the path to `swiftc`
+# in order to compile.
+swift_exec = [os.getenv('SWIFT_EXEC')]
+assert swift_exec[0] is not None, \
+    'Environment variable $SWIFT_EXEC must be set.'
+
+if platform.system() == 'Darwin':
+    # On OS X, we need to make sure swiftc references the
+    # proper SDK, has a deployment target set, and more...
+    # Here we rely on environment variables, produced by xcodebuild.
+    sdk_root = os.getenv('SDKROOT')
+    built_products_dir = os.getenv('BUILT_PRODUCTS_DIR')
+    platform_name = os.getenv('PLATFORM_NAME')
+    deployment_target = os.getenv('MACOSX_DEPLOYMENT_TARGET')
+    assert sdk_root and built_products_dir and platform_name and deployment_target, \
+        'Environment variables $SDKROOT, $BUILT_PRODUCTS_DIR, ' + \
+        '$PLATFORM_NAME, and $MACOSX_DEPLOYMENT_TARGET must all be set.\n' + \
+        '$SDKROOT: "{sdk_root}"\n' + \
+        '$BUILT_PRODUCTS_DIR: "{built_products_dir}"\n' + \
+        '$PLATFORM_NAME: "{platform_name}"\n' + \
+        '$MACOSX_DEPLOYMENT_TARGET: "{deployment_target}"\n'.format(
+            sdk_root=sdk_root, built_products_dir=built_products_dir,
+            platform_name=platform_name, deployment_target=deployment_target)
+
+    target = '{}-apple-{}{}'.format(
+        platform.machine(), platform_name, deployment_target)
+    swift_exec.extend([
+        '-sdk', sdk_root,
+        '-target', target,
+        '-L', built_products_dir,
+        '-I', built_products_dir,
+        '-F', built_products_dir,
+        '-Xlinker', '-rpath',
+        '-Xlinker', built_products_dir])
+
+# Having prepared the swiftc command, we set the substitution.
+config.substitutions.append(('%swift_exec', ' '.join(swift_exec)))
+
+# Add the %built_tests_dir substitution, which is a temporary
+# directory used to store built files.
+built_tests_dir = tempfile.mkdtemp()
+config.substitutions.append(('%built_tests_dir', built_tests_dir))
+
+# Add the %test_output substitution, which is a temporary file
+# used to store test output.
+test_output = tempfile.mkstemp()[1]
+config.substitutions.append(('%test_output', test_output))
+
+# Add the %xctest-checker substitution, which is a Python script
+# can be used to compare the actual XCTest output to the expected
+# output.
+xctest_checker = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'xctest_checker',
+    'xctest_checker.py')
+config.substitutions.append(('%xctest-checker', xctest_checker))

Tests/Functional/xctest_checker/.gitignore

@@ -0,0 +1,26 @@
+# Compile artifacts
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution/packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt

Tests/Functional/xctest_checker/setup.py

@@ -0,0 +1,43 @@
+import os
+import setuptools
+
+import xctest_checker
+
+# setuptools expects to be invoked from within the directory of setup.py,
+# but it is nice to allow `python path/to/setup.py install` to work
+# (for scripts, etc.)
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+setuptools.setup(
+    name='xctest_checker',
+    version=xctest_checker.__version__,
+
+    author=xctest_checker.__author__,
+    author_email=xctest_checker.__email__,
+    url='http://swift.org',
+    license='Apache',
+
+    description="A tool to verify the output of XCTest runs.",
+    keywords='test xctest swift',
+
+    test_suite='tests',
+
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Testing',
+    ],
+
+    zip_safe=False,
+    packages=setuptools.find_packages(),
+    entry_points={
+        'console_scripts': [
+            'xctest_checker = xctest_checker:main',
+        ],
+    }
+)

Tests/Functional/xctest_checker/tests/__init__.py

@@ -0,0 +1,29 @@
+import tempfile
+import unittest
+
+from xctest_checker import compare
+
+
+def _tmpfile(content):
+    """Returns the path to a temp file with the given contents."""
+    tmp = tempfile.mkstemp()[1]
+    with open(tmp, 'w') as f:
+        f.write(content)
+    return tmp
+
+
+class CompareTestCase(unittest.TestCase):
+    def test_no_match_raises(self):
+        actual = _tmpfile('foo\nbar\nbaz\n')
+        expected = _tmpfile('c: foo\nc: baz\nc: bar\n')
+        with self.assertRaises(AssertionError):
+            compare.compare(actual, expected, check_prefix='c: ')
+
+    def test_match_does_not_raise(self):
+        actual = _tmpfile('foo\nbar\nbaz\n')
+        expected = _tmpfile('c: foo\nc: bar\ndoes not match but not checked\n')
+        compare.compare(actual, expected, check_prefix='c: ')
+
+
+if __name__ == "__main__":
+    unittest.main()

Tests/Functional/xctest_checker/tests/test_compare.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import xctest_checker
+
+if __name__ == '__main__':
+    xctest_checker.main()

Tests/Functional/xctest_checker/xctest_checker.py

@@ -0,0 +1,9 @@
+from __future__ import absolute_import
+from .main import main
+
+__author__ = 'Brian Gesiak'
+__email__ = '[email protected]'
+__versioninfo__ = (0, 1, 0)
+__version__ = '.'.join(str(v) for v in __versioninfo__)
+
+__all__ = []

Tests/Functional/xctest_checker/xctest_checker/__init__.py

@@ -0,0 +1,38 @@
+import re
+
+
+def _actual_lines(path):
+    """
+    Returns a generator that yields each line in the file at the given path.
+    """
+    with open(path) as f:
+        for line in f:
+            yield line
+
+
+def _expected_lines(path, check_prefix):
+    """
+    Returns a generator that yields each line in the file at the given path
+    that begins with the given prefix.
+    """
+    with open(path) as f:
+        for line in f:
+            if line.startswith(check_prefix):
+                yield line[len(check_prefix):]
+
+
+def compare(actual, expected, check_prefix):
+    """
+    Compares each line in the two given files.
+    If any line in the 'actual' file doesn't match the regex in the 'expected'
+    file, raises an AssertionError.
+    """
+    for lines in zip(
+            _actual_lines(actual),
+            _expected_lines(expected, check_prefix)):
+        if not re.match(lines[1], lines[0]):
+            raise AssertionError('Actual line did not match the expected '
+                                 'regular expression.\n'
+                                 'Actual: {}\n'
+                                 'Expected: {}\n'.format(
+                                     repr(lines[0]), repr(lines[1])))

Tests/Functional/xctest_checker/xctest_checker/compare.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+
+import argparse
+
+from . import compare
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('actual', help='A path to a file containing the '
+                                       'actual output of an XCTest run.')
+    parser.add_argument('expected', help='A path to a file containing the '
+                                         'expected output of an XCTest run.')
+    parser.add_argument('-p', '--check-prefix',
+                        default='// CHECK: ',
+                        help='{prog} checks actual output against expected '
+                             'output. By default, {prog} only checks lines '
+                             'that are prefixed with "// CHECK: ". This '
+                             'option can be used to change that '
+                             'prefix.'.format(prog=parser.prog))
+    args = parser.parse_args()
+    compare.compare(args.actual, args.expected, args.check_prefix)
+
+
+if __name__ == '__main__':
+    main()