Merge pull request #15211 from davezarzycki/formalize_stress_tests

[Testing] Formalize stress tests

Author: davezarzycki

docs/Testing.md

@@ -33,6 +33,7 @@ The testsuite is split into four subsets:
 * Validation testsuite, located under ``swift/validation-test``.
 * Unit tests, located under ``swift/unittests``.
 * Long tests, which are marked with ``REQUIRES: long_test``.
+* Stress tests, which are marked with ``REQUIRES: stress_test``.
 
 ### Running the LLVM lit-based testsuite
 
@@ -126,8 +127,9 @@ Besides ``check-swift``, other targets are also available. Here's the full list:
 
 * ``check-swift``: Runs tests from the ``${SWIFT_SOURCE_ROOT}/test`` directory.
 * ``check-swift-only_validation``: Runs tests from the ``${SWIFT_SOURCE_ROOT}/validation-test`` directory.
-* ``check-swift-validation``: Runs the primary and validation tests, without the long tests.
+* ``check-swift-validation``: Runs the primary and validation tests, without the long tests or stress tests.
 * ``check-swift-only_long``: Runs long tests only.
+* ``check-swift-only_stress``: Runs stress tests only.
 * ``check-swift-all``: Runs all tests (primary, validation, and long).
 * ``SwiftUnitTests``: Builds all unit tests.  Executables are located under
   ``${SWIFT_BUILD_DIR}/unittests`` and must be run individually.

test/CMakeLists.txt

@@ -146,6 +146,7 @@ set(TEST_SUBSETS
     all
     only_validation
     only_long
+    only_stress
 )
 
 if(NOT "${COVERAGE_DB}" STREQUAL "")
@@ -271,12 +272,14 @@ foreach(SDK ${SWIFT_SDKS})
         if((test_subset STREQUAL "primary") OR
            (test_subset STREQUAL "validation") OR
            (test_subset STREQUAL "only_long") OR
+           (test_subset STREQUAL "only_stress") OR
            (test_subset STREQUAL "all"))
           list(APPEND directories "${test_bin_dir}")
         endif()
         if((test_subset STREQUAL "validation") OR
            (test_subset STREQUAL "only_validation") OR
            (test_subset STREQUAL "only_long") OR
+           (test_subset STREQUAL "only_stress") OR
            (test_subset STREQUAL "all"))
           list(APPEND directories "${validation_test_bin_dir}")
           list(APPEND dependencies ${validation_test_dependencies})

test/Unit/lit.cfg

@@ -58,6 +58,10 @@ elif swift_test_subset == 'only_long':
     # FIXME: this doesn't exclude not-long tests from the only_long subset.
     #   Currently those tests are very fast so it doesn't matter much.
     pass
+elif swift_test_subset == 'only_stress':
+    # FIXME: this doesn't exclude not-stress tests from the only_stress subset.
+    #   Currently those tests are very fast so it doesn't matter much.
+    pass
 else:
     lit_config.fatal("Unknown test mode %r" % swift_test_subset)
 

test/lit.cfg

@@ -504,10 +504,15 @@ if swift_test_subset in ['primary', 'validation', 'only_validation']:
     pass
 elif swift_test_subset == 'all':
     config.available_features.add("long_test")
+    config.available_features.add("stress_test")
 elif swift_test_subset == 'only_long':
     config.available_features.add("long_test")
     config.limit_to_features.add("long_test")
     config.limit_to_features.discard("executable_test")
+elif swift_test_subset == 'only_stress':
+    config.available_features.add("stress_test")
+    config.limit_to_features.add("stress_test")
+    config.limit_to_features.discard("executable_test")
 else:
     lit_config.fatal("Unknown test mode %r" % swift_test_subset)
 

utils/build-presets.ini

@@ -236,6 +236,7 @@ mixin-preset=
 test=0
 validation-test=0
 long-test=1
+stress-test=1
 test-optimized=0
 
 
@@ -747,6 +748,7 @@ release
 test
 validation-test
 long-test
+stress-test
 test-optimized
 foundation
 libdispatch
@@ -822,6 +824,7 @@ release
 test
 validation-test
 long-test
+stress-test
 foundation
 lit-args=-v
 
@@ -892,6 +895,7 @@ mixin-preset=buildbot_incremental_linux
 test=0
 validation-test=0
 long-test=1
+stress-test=1
 test-optimized=0
 
 [preset: buildbot_incremental_linux,llvm-only]
@@ -901,6 +905,7 @@ build-subdir=buildbot_incremental_llvmonly
 test=0
 validation-test=0
 long-test=0
+stress-test=0
 test-optimized=0
 
 dash-dash
@@ -1022,6 +1027,7 @@ watchos
 test
 validation-test
 long-test
+stress-test
 
 dash-dash
 

utils/build-script

@@ -154,11 +154,12 @@ class HostSpecificConfiguration(object):
                     test = False
 
             if build:
-                # Validation and long tests require building the full standard
-                # library, whereas the other targets can build a slightly
-                # smaller subset which is faster to build.
+                # Validation, long, and stress tests require building the full
+                # standard library, whereas the other targets can build a
+                # slightly smaller subset which is faster to build.
                 if args.build_swift_stdlib_unittest_extra or \
-                        args.validation_test or args.long_test:
+                        args.validation_test or args.long_test or \
+                        args.stress_test:
                     self.swift_stdlib_build_targets.append(
                         "swift-stdlib-" + name)
                 else:
@@ -184,12 +185,15 @@ class HostSpecificConfiguration(object):
                 else:
                     suffix = ""
                 subset_suffix = ""
-                if args.validation_test and args.long_test:
+                if args.validation_test and args.long_test and \
+                        args.stress_test:
                     subset_suffix = "-all"
                 elif args.validation_test:
                     subset_suffix = "-validation"
                 elif args.long_test:
                     subset_suffix = "-only_long"
+                elif args.stress_test:
+                    subset_suffix = "-only_stress"
                 else:
                     subset_suffix = ""
                 self.swift_test_run_targets.append("check-swift{}{}-{}".format(
@@ -635,7 +639,7 @@ class BuildScriptInvocation(object):
         if not args.build_android:
             impl_args += ["--skip-build-android"]
 
-        if not args.test and not args.long_test:
+        if not args.test and not args.long_test and not args.stress_test:
             impl_args += ["--skip-test-swift"]
         if not args.test:
             impl_args += ["--skip-test-cmark",
@@ -677,6 +681,8 @@ class BuildScriptInvocation(object):
             impl_args += ["--validation-test"]
         if args.long_test:
             impl_args += ["--long-test"]
+        if args.stress_test:
+            impl_args += ["--stress-test"]
         if not args.benchmark:
             impl_args += ["--skip-test-benchmarks"]
         if not args.test_optimized:

utils/build-script-impl

@@ -153,6 +153,7 @@ KNOWN_SETTINGS=(
     skip-test-android-host      ""               "set to skip testing the host parts of the Android toolchain"
     validation-test             "0"              "set to run the validation test suite"
     long-test                   "0"              "set to run the long test suite"
+    stress-test                 "0"              "set to run the stress test suite"
     test-paths                  ""               "run tests located in specific directories and/or files"
     skip-test-benchmarks        ""               "set to skip running Swift Benchmark Suite"
     skip-test-optimized         ""               "set to skip testing the test suite in optimized mode"

utils/build_swift/driver_arguments.py

@@ -703,6 +703,9 @@ def create_argument_parser():
     option('--long-test', toggle_true,
            help='run the long test suite')
 
+    option('--stress-test', toggle_true,
+           help='run the stress test suite')
+
     option('--host-test', toggle_true,
            help='run executable tests on host devices (such as iOS or tvOS)')
 

utils/build_swift/tests/expected_options.py

@@ -149,6 +149,7 @@
     'show_sdks': False,
     'skip_build': False,
     'stdlib_deployment_targets': None,
+    'stress_test': False,
     'swift_analyze_code_coverage': defaults.SWIFT_ANALYZE_CODE_COVERAGE,
     'swift_assertions': True,
     'swift_build_variant': 'Debug',
@@ -423,6 +424,7 @@ class IgnoreOption(_BaseOption):
     EnableOption('--libicu', dest='build_libicu'),
     EnableOption('--long-test'),
     EnableOption('--show-sdks'),
+    EnableOption('--stress-test'),
     EnableOption('--test'),
     EnableOption('--test-optimize-for-size'),
     EnableOption('--test-optimized'),

utils/build_swift/tests/test_driver_arguments.py

@@ -645,6 +645,7 @@ def test_implied_defaults_skip_all_tests(self):
                 '--test', '0',
                 '--validation-test', '0',
                 '--long-test', '0',
+                '--stress-test', '0',
             ])
 
             self.assertFalse(namespace.test_linux)

utils/run-test

@@ -42,6 +42,7 @@ TEST_SUBSETS = [
     'all',
     'only_validation',
     'only_long',
+    'only_stress',
 ]
 
 SWIFT_SOURCE_DIR = os.path.join(SWIFT_SOURCE_ROOT, 'swift')

test/IDE/print_stdlib_specialized.swift renamed to validation-test/IDE/print_stdlib_specialized.swift

@@ -1,6 +1,6 @@
 // RUN: %target-run-simple-swift
 // REQUIRES: executable_test
-
+// REQUIRES: stress_test
 // REQUIRES: objc_interop
 
 import StdlibUnittest

test/Runtime/weak-reference-racetests-dispatch.swift renamed to validation-test/Runtime/weak-reference-racetests-dispatch.swift

@@ -1,5 +1,6 @@
 // RUN: %target-run-simple-swift
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 import StdlibUnittest

test/Runtime/weak-reference-racetests.swift renamed to validation-test/Runtime/weak-reference-racetests.swift

@@ -2,6 +2,7 @@
 // RUN: %target-build-swift -O %s -module-name=test -Xllvm -sil-disable-pass=FunctionSignatureOpts -emit-sil | %FileCheck %s
 // RUN: %target-run %t.out
 // REQUIRES: swift_stdlib_no_asserts,optimized_stdlib
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 import StdlibUnittest

test/SILOptimizer/string_switch.swift renamed to validation-test/SILOptimizer/string_switch.swift

@@ -1,6 +1,7 @@
 // RUN: %target-build-swift -target %sanitizers-target-triple -sanitize=thread %s -o %t_binary
 // RUN: %env-TSAN_OPTIONS=ignore_interceptors_accesses=1:halt_on_error=1 %target-run %t_binary
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // REQUIRES: tsan_runtime
 // REQUIRES: objc_interop
 

test/Sanitizers/Inputs/tsan-uninstrumented.swift renamed to validation-test/Sanitizers/Inputs/tsan-uninstrumented.swift

@@ -6,6 +6,7 @@
 // RUN: not env %env-TSAN_OPTIONS=abort_on_error=0 %target-run %t/tsan-binary 2>&1 | %FileCheck %s
 // RUN: not env %env-TSAN_OPTIONS=abort_on_error=0:ignore_interceptors_accesses=0 %target-run %t/tsan-binary 2>&1 | %FileCheck %s --check-prefix CHECK-INTERCEPTORS-ACCESSES
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // REQUIRES: objc_interop
 // REQUIRES: tsan_runtime
 

test/Sanitizers/tsan-ignores-arc-locks.swift renamed to validation-test/Sanitizers/tsan-ignores-arc-locks.swift

@@ -1,6 +1,7 @@
 // RUN: %target-swiftc_driver -target %sanitizers-target-triple -sanitize=thread %s -o %t_binary
 // RUN: %env-TSAN_OPTIONS=ignore_interceptors_accesses=1:halt_on_error=1 %target-run %t_binary
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // REQUIRES: objc_interop
 // REQUIRES: tsan_runtime
 

test/Sanitizers/tsan-inout.swift renamed to validation-test/Sanitizers/tsan-inout.swift

@@ -1,6 +1,7 @@
 // RUN: %target-build-swift -sanitize=thread -target %sanitizers-target-triple %s -o %t_binary
 // RUN: %env-TSAN_OPTIONS=ignore_interceptors_accesses=1:halt_on_error=1 %target-run %t_binary
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // REQUIRES: tsan_runtime
 
 // https://bugs.swift.org/browse/SR-6622

test/Sanitizers/tsan-type-metadata.swift renamed to validation-test/Sanitizers/tsan-type-metadata.swift

@@ -1,5 +1,6 @@
 // RUN: %target-build-swift -Xfrontend -disable-access-control -module-name a %s -o %t.out
 // RUN: %target-run %t.out | %FileCheck %s
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 import StdlibUnittest

test/Sanitizers/witness_table_lookup.swift renamed to validation-test/Sanitizers/witness_table_lookup.swift

@@ -8,6 +8,7 @@
 
 // REQUIRES: objc_interop
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 import StdlibUnittest

validation-test/StdlibUnittest/RaceTest.swift

@@ -3,6 +3,7 @@
 // RUN: %target-build-swift -module-name a %s -o %t.out -O
 // RUN: %target-run %t.out
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 import SwiftPrivate

validation-test/stdlib/ArrayBridging.swift

@@ -6,6 +6,7 @@
 // RUN: %target-build-swift %t/CommandLineStressTest.o %t/CommandLineStressTestSwift.o -o %t/CommandLineStressTest
 // RUN: %target-run %t/CommandLineStressTest foo bar baz qux quux corge grault garply waldo fred plugh xyzzy and thud
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // UNSUPPORTED: nonatomic_rc
 
 // This file is an empty stub to call into the command line stress test which

validation-test/stdlib/AtomicInt.swift

@@ -5,6 +5,7 @@
 // RUN: %target-build-swift -Xfrontend -disable-access-control -I %S/Inputs/SlurpFastEnumeration/ %t/main.swift %S/Inputs/DictionaryKeyValueTypes.swift %S/Inputs/DictionaryKeyValueTypesObjC.swift -Xlinker %t/SlurpFastEnumeration.o -o %t.out -O
 // RUN: %target-run %t.out
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 
 // REQUIRES: objc_interop
 // UNSUPPORTED: nonatomic_rc

test/stdlib/CommandLine.swift renamed to validation-test/stdlib/CommandLine.swift

@@ -1,5 +1,6 @@
 // RUN: %target-run-simple-swift
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 // REQUIRES: objc_interop
 // UNSUPPORTED: nonatomic_rc
 

validation-test/stdlib/DictionaryBridging.swift

@@ -1,5 +1,6 @@
 // RUN: %target-run-simple-swift
 // REQUIRES: executable_test
+// REQUIRES: stress_test
 
 import StdlibUnittest
 import SwiftPrivatePthreadExtras