[test] Add '--param swift-version=3' to lit's configuration. (swiftlang#5566)

This allows us to run our entire test suite (well, okay, just
invocations of swift/swiftc and swift-ide-test) under Swift 3 mode or
Swift 4 mode, which will be more important as the two modes diverge.
The default is Swift 3 mode for now, which matches the behavior before
this patch (because the master compiler still calls itself 3.0).

Individual tests can still use "-swift-version 3" to override the mode
set by lit, but if an entire test requires Swift 3 or Swift 4, there's
also a new test feature "SWIFT_VERSION=3" or "SWIFT_VERSION=4".
However, if you /do/ need to override or restrict the version, you
should also add a test for the "other" version as well, unless it's
part of the Compatibility suite or otherwise specifically testing
-swift-version-related functionality.

Author: jrose-apple

docs/Testing.rst

@@ -150,6 +150,23 @@ targets mentioned above and modify it as necessary. lit.py also has several
 useful features, like timing tests and providing a timeout. Check these features
 out with ``lit.py -h``.
 
+Extra lit.py invocation options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* ``--param gmalloc`` will run all tests under Guard Malloc (macOS only). See
+  ``man libgmalloc`` for more information.
+
+* ``--param swift-version=<MAJOR>`` overrides the default Swift language
+  version used by swift/swiftc and swift-ide-test.
+
+* ``--param interpret`` is an experimental option for running execution tests
+  using Swift's interpreter rather than compiling them first. Note that this
+  does not affect all substitutions.
+
+* ``--param swift_test_mode=<MODE>`` drives the various suffix variations
+  mentioned above. Again, it's best to get the invocation from the existing
+  build system targets and modify it rather than constructing it yourself.
+
 Writing tests
 -------------
 
@@ -422,6 +439,10 @@ FIXME: full list.
 
 * ``optimized_stdlib_<CPUNAME>``: an optimized stdlib plus cpu configuration.
 
+* ``SWIFT_VERSION=<MAJOR>``: restricts a test to Swift 3 or Swift 4. If you
+  need to use this, make sure to add a test for the other version as well
+  unless you are specifically testing ``-swift-version``-related functionality.
+
 * ``XFAIL: linux``: tests that need to be adapted for Linux, for example parts
   that depend on Objective-C interop need to be split out.
 

test/ClangModules/invalid_bridging_header.swift

@@ -1,4 +1,4 @@
-// RUN: not %target-swift-frontend-parse -import-objc-header %S/Inputs/invalid_bridging_header.h %s > %t.out 2>&1 
+// RUN: not %target-swift-frontend -parse -import-objc-header %S/Inputs/invalid_bridging_header.h %s > %t.out 2>&1 
 // RUN: %FileCheck %s < %t.out
 
 // REQUIRES: objc_interop

test/Compatibility/attr_escaping.swift

@@ -1,4 +1,5 @@
 // RUN: %target-parse-verify-swift
+// REQUIRES: SWIFT_VERSION=3
 
 // This is allowed, in order to keep source compat with Swift version 3.0.
 func takesVarargsOfFunctionsExplicitEscaping(_ fns: @escaping () -> ()...) {} 

test/Compatibility/protocol_composition.swift

@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: %utils/split_file.py -o %t %s
-// RUN: %target-swift-frontend -parse -primary-file %t/swift3.swift %t/common.swift -verify
+// RUN: %target-swift-frontend -parse -primary-file %t/swift3.swift %t/common.swift -verify -swift-version 3
 // RUN: %target-swift-frontend -parse -primary-file %t/swift4.swift %t/common.swift -verify -swift-version 4
 
 // BEGIN common.swift

test/Driver/swift-version-default.swift

@@ -0,0 +1,17 @@
+// RUN: %swiftc_driver_plain -parse -Xfrontend -verify %s
+
+// This test should be updated to match the expected default Swift version
+// when swiftc is invoked directly.
+// It should /not/ follow the version specified when invoking lit.
+
+#if swift(>=3)
+asdf // expected-error {{use of unresolved identifier}}
+#else
+jkl
+#endif
+
+#if swift(>=4)
+aoeu
+#else
+htn // expected-error {{use of unresolved identifier}}
+#endif

test/lit.cfg

@@ -323,8 +323,18 @@ if test_sdk_overlay_dir is not None:
     lit_config.note('Using SDK overlay dir: ' + test_sdk_overlay_dir)
     resource_dir_opt += (" %s" % sdk_overlay_dir_opt)
 
+# Default to Swift 3 for now.
+# Note that this accepts both `--param swift-version` (like the compiler flag)
+# and `--param swift_version` (like a lit configuration parameter).
+swift_version = lit_config.params.get('swift-version',
+    lit_config.params.get('swift_version', '3'))
+lit_config.note('Compiling with -swift-version ' + swift_version)
+config.swift_test_options = '-swift-version ' + swift_version
+
 test_options = os.environ.get('SWIFT_TEST_OPTIONS')
-config.swift_test_options = ' ' + test_options if test_options else ''
+if test_options:
+    config.swift_test_options += ' '
+    config.swift_test_options += test_options
 
 clang_module_cache_path = tempfile.mkdtemp(prefix="swift-testsuite-clang-module-cache")
 mcp_opt = "-module-cache-path %r" % clang_module_cache_path
@@ -341,22 +351,23 @@ config.substitutions.append( ('%{python}', sys.executable) )
 config.substitutions.append( ('%mcp_opt', mcp_opt) )
 config.substitutions.append( ('%swift_driver_plain', "%r" % config.swift) )
 config.substitutions.append( ('%swiftc_driver_plain', "%r" % config.swiftc) )
-config.substitutions.append( ('%swift_driver', "env SDKROOT= %r %s%s" % (config.swift, mcp_opt, config.swift_test_options)) )
-config.substitutions.append( ('%swiftc_driver', "env SDKROOT= %r %s%s" % (config.swiftc, mcp_opt, config.swift_test_options)) )
+config.substitutions.append( ('%swift_driver', "env SDKROOT= %r %s %s" % (config.swift, mcp_opt, config.swift_test_options)) )
+config.substitutions.append( ('%swiftc_driver', "env SDKROOT= %r %s %s" % (config.swiftc, mcp_opt, config.swift_test_options)) )
 config.substitutions.append( ('%sil-opt', "%r %s" % (config.sil_opt, mcp_opt)) )
 config.substitutions.append( ('%sil-extract', "%r %s" % (config.sil_extract, mcp_opt)) )
 config.substitutions.append( ('%lldb-moduleimport-test', "%r %s" % (config.lldb_moduleimport_test, mcp_opt)) )
 config.substitutions.append( ('%swift-ide-test_plain', config.swift_ide_test) )
-config.substitutions.append( ('%swift-ide-test', "%r %s %s" % (config.swift_ide_test, mcp_opt, ccp_opt)) )
+config.substitutions.append( ('%swift-ide-test', "%r %s %s -swift-version %s" % (config.swift_ide_test, mcp_opt, ccp_opt, swift_version)) )
 config.substitutions.append( ('%swift-format', config.swift_format) )
 config.substitutions.append( ('%llvm-link', config.llvm_link) )
 config.substitutions.append( ('%swift-llvm-opt', config.swift_llvm_opt) )
 config.substitutions.append( ('%llvm-dwarfdump', config.llvm_dwarfdump) )
 
 # This must come after all substitutions containing "%swift".
 config.substitutions.append(
-    ('%swift', "%r -frontend %s -disable-objc-attr-requires-foundation-module%s"
-     % (config.swift, mcp_opt, config.swift_test_options)) )
+    ('%swift',
+     "%r -frontend %s -disable-objc-attr-requires-foundation-module %s"
+       % (config.swift, mcp_opt, config.swift_test_options)) )
 
 config.clang_include_dir = \
   os.path.join(os.path.dirname(os.path.dirname(config.swift)), 'include')
@@ -443,6 +454,8 @@ config.available_features.add("PTRSIZE=" + run_ptrsize)
 if run_cpu == "i386" or run_cpu == "x86_64":
   config.available_features.add("CPU=i386_or_x86_64")
 
+config.available_features.add("SWIFT_VERSION=" + swift_version)
+
 if "optimized_stdlib" in config.available_features:
   config.available_features.add("optimized_stdlib_" + run_cpu)
 
@@ -736,7 +749,7 @@ elif run_os in ['linux-gnu', 'linux-gnueabihf', 'freebsd', 'windows-cygnus', 'wi
     config.target_run = ''
     if 'interpret' in lit_config.params:
         target_run_base = (
-            '%s -target %s %s %s -module-name main%s %s'
+            '%s -target %s %s %s -module-name main %s %s'
             % (config.swift, config.variant_triple, resource_dir_opt,
                mcp_opt, config.swift_test_options,
                swift_execution_tests_extra_flags))