Skip to content

Commit 039331c

Browse files
shahmishalshahmishal
committed
Merge pull request #2213 from apple/build-script-long-tests
CMake & build-script: add a new tier of testing, "long tests"
2 parents 5533880 + 457f2b9 commit 039331c

25 files changed

+264
-161
lines changed

‎docs/Testing.rst

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,17 @@ The LLVM lit-based testsuite
3333

3434
* Buildbots run all tests, on all supported platforms.
3535

36+
Testsuite subsets
37+
-----------------
38+
39+
The testsuite is split into three subsets:
40+
41+
* Primary testsuite, located under ``swift/test``.
42+
43+
* Validation testsuite, located under ``swift/validation-test``.
44+
45+
* Long tests, which are marked with ``REQUIRES: long_test``.
46+
3647
Running the LLVM lit-based testsuite
3748
------------------------------------
3849

@@ -59,13 +70,21 @@ Besides ``check-swift``, other targets are also available. Here's the full list:
5970

6071
Runs tests from the ``${SWIFT_SOURCE_ROOT}/test`` directory.
6172

62-
* ``check-swift-validation``
73+
* ``check-swift-only_validation``
6374

6475
Runs tests from the ``${SWIFT_SOURCE_ROOT}/validation-test`` directory.
6576

77+
* ``check-swift-validation``
78+
79+
Runs the primary and validation tests, without the long tests.
80+
81+
* ``check-swift-only_long``
82+
83+
Runs long tests only.
84+
6685
* ``check-swift-all``
6786

68-
Runs all tests.
87+
Runs all tests (primary, validation, and long).
6988

7089
For every target above, there are variants for different optimizations:
7190

‎test/CMakeLists.txt

Lines changed: 147 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -117,53 +117,109 @@ if(PYTHONINTERP_FOUND)
117117
set(SWIFT_ASAN_BUILD TRUE)
118118
endif()
119119

120-
set(TEST_MODES optimize_none optimize optimize_unchecked executable non_executable)
120+
# Normalize spelling of boolean values.
121+
normalize_boolean_spelling(LLVM_ENABLE_ASSERTIONS)
122+
normalize_boolean_spelling(SWIFT_STDLIB_ASSERTIONS)
123+
normalize_boolean_spelling(SWIFT_AST_VERIFIER)
124+
normalize_boolean_spelling(SWIFT_ASAN_BUILD)
125+
is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED)
126+
127+
if(SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "MERGED")
128+
set(profdata_merge_worker
129+
"${CMAKE_CURRENT_SOURCE_DIR}/../utils/profdata_merge/main.py")
130+
set(command_profdata_merge_start
131+
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}"
132+
-l "${swift_test_results_dir}/profdata_merge.log"
133+
start
134+
-o "${swift_test_results_dir}")
135+
set(command_profdata_merge_stop
136+
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" stop)
137+
else()
138+
set(command_profdata_merge_start)
139+
set(command_profdata_merge_stop)
140+
endif()
141+
142+
set(TEST_MODES
143+
optimize_none optimize optimize_unchecked
144+
only_executable only_non_executable
145+
)
146+
set(TEST_SUBSETS
147+
primary
148+
validation
149+
all
150+
only_validation
151+
only_long
152+
)
121153

122154
foreach(SDK ${SWIFT_SDKS})
123155
foreach(ARCH ${SWIFT_SDK_${SDK}_ARCHITECTURES})
124-
foreach(TEST_MODE ${TEST_MODES})
125-
# Configure variables for this subdirectory.
126-
set(VARIANT_SUFFIX "-${SWIFT_SDK_${SDK}_LIB_SUBDIR}-${ARCH}")
127-
set(VARIANT_TRIPLE "${SWIFT_SDK_${SDK}_ARCH_${ARCH}_TRIPLE}")
128-
set(VARIANT_SDK "${SWIFT_SDK_${SDK}_PATH}")
129-
is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED)
130-
131-
# Normalize spelling of boolean values.
132-
normalize_boolean_spelling(LLVM_ENABLE_ASSERTIONS)
133-
normalize_boolean_spelling(SWIFT_STDLIB_ASSERTIONS)
134-
normalize_boolean_spelling(SWIFT_AST_VERIFIER)
135-
normalize_boolean_spelling(SWIFT_ASAN_BUILD)
136-
137-
# A directory where to put the xUnit-style XML test results.
138-
set(swift_test_results_dir
139-
"${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/swift-test-results/${VARIANT_TRIPLE}")
140-
141-
set(command_clean_test_results_dir
142-
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${swift_test_results_dir}"
143-
COMMAND "${CMAKE_COMMAND}" -E make_directory "${swift_test_results_dir}")
156+
# Configure variables for this subdirectory.
157+
set(VARIANT_SUFFIX "-${SWIFT_SDK_${SDK}_LIB_SUBDIR}-${ARCH}")
158+
set(VARIANT_TRIPLE "${SWIFT_SDK_${SDK}_ARCH_${ARCH}_TRIPLE}")
159+
set(VARIANT_SDK "${SWIFT_SDK_${SDK}_PATH}")
160+
161+
# A directory where to put the xUnit-style XML test results.
162+
set(swift_test_results_dir
163+
"${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/swift-test-results/${VARIANT_TRIPLE}")
164+
165+
set(command_clean_test_results_dir
166+
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${swift_test_results_dir}"
167+
COMMAND "${CMAKE_COMMAND}" -E make_directory "${swift_test_results_dir}")
168+
169+
set(test_bin_dir "${CMAKE_CURRENT_BINARY_DIR}${VARIANT_SUFFIX}")
170+
set(validation_test_bin_dir
171+
"${CMAKE_CURRENT_BINARY_DIR}/../validation-test${VARIANT_SUFFIX}")
172+
173+
swift_configure_lit_site_cfg(
174+
"${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in"
175+
"${test_bin_dir}/lit.site.cfg"
176+
"test${VARIANT_SUFFIX}.lit.site.cfg")
177+
178+
swift_configure_lit_site_cfg(
179+
"${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in"
180+
"${test_bin_dir}/Unit/lit.site.cfg"
181+
"")
182+
183+
swift_configure_lit_site_cfg(
184+
"${CMAKE_CURRENT_SOURCE_DIR}/../validation-test/lit.site.cfg.in"
185+
"${validation_test_bin_dir}/lit.site.cfg"
186+
"validation-test${VARIANT_SUFFIX}.lit.site.cfg")
187+
188+
set(test_dependencies)
189+
get_test_dependencies("${SDK}" test_dependencies)
190+
list(APPEND test_dependencies
191+
"swift-test-stdlib-${SWIFT_SDK_${SDK}_LIB_SUBDIR}")
192+
193+
if(SWIFT_BUILD_STDLIB AND SWIFT_INCLUDE_TESTS)
194+
list(APPEND test_dependencies
195+
"swift-reflection-test${VARIANT_SUFFIX}")
196+
endif()
144197

198+
set(validation_test_dependencies
199+
"swiftStdlibCollectionUnittest-${SWIFT_SDK_${SDK}_LIB_SUBDIR}")
200+
201+
foreach(test_mode ${TEST_MODES})
145202
set(LIT_ARGS "${SWIFT_TEST_EXTRA_ARGS} ${LLVM_LIT_ARGS}")
146203
separate_arguments(LIT_ARGS)
147204

205+
if(test_mode STREQUAL "optimize_none")
206+
# No special arguments required.
207+
elseif(test_mode STREQUAL "optimize")
208+
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
209+
elseif(test_mode STREQUAL "optimize_unchecked")
210+
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
211+
elseif(test_mode STREQUAL "only_executable")
212+
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
213+
elseif(test_mode STREQUAL "only_non_executable")
214+
list(APPEND LIT_ARGS "--param" "run_only_tests=non_executable_tests")
215+
else()
216+
message(FATAL_ERROR "Unknown test mode: ${test_mode}")
217+
endif()
218+
148219
set(test_mode_target_suffix "")
149-
if(NOT TEST_MODE STREQUAL "optimize_none")
150-
if(TEST_MODE STREQUAL "optimize")
151-
set(test_mode_target_suffix "-optimize")
152-
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
153-
endif()
154-
if(TEST_MODE STREQUAL "optimize_unchecked")
155-
set(test_mode_target_suffix "-optimize-unchecked")
156-
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
157-
endif()
158-
if(TEST_MODE STREQUAL "executable")
159-
set(test_mode_target_suffix "-executable")
160-
list(APPEND LIT_ARGS "--param" "run_only_tests=executable_tests")
161-
endif()
162-
if(TEST_MODE STREQUAL "non_executable")
163-
set(test_mode_target_suffix "-non-executable")
164-
list(APPEND LIT_ARGS "--param" "run_only_tests=non_executable_tests")
165-
endif()
166-
list(APPEND LIT_ARGS "--param" "swift_test_mode=${TEST_MODE}")
220+
if(NOT test_mode STREQUAL "optimize_none")
221+
list(APPEND LIT_ARGS "--param" "swift_test_mode=${test_mode}")
222+
set(test_mode_target_suffix "-${test_mode}")
167223
endif()
168224

169225
if(NOT SWIFT_BUILD_STDLIB)
@@ -173,123 +229,74 @@ if(PYTHONINTERP_FOUND)
173229

174230
list(APPEND LIT_ARGS "--xunit-xml-output=${swift_test_results_dir}/lit-tests.xml")
175231

176-
set(lit_command ${PYTHON_EXECUTABLE} "${LIT}" ${LIT_ARGS})
177-
178232
set(command_upload_stdlib)
179233
if("${SDK}" STREQUAL "IOS" OR "${SDK}" STREQUAL "TVOS" OR "${SDK}" STREQUAL "WATCHOS")
180234
# These are supported testing SDKs.
181235
endif()
182236

183-
set(test_dependencies)
184-
get_test_dependencies("${SDK}" test_dependencies)
185-
list(APPEND test_dependencies
186-
"swift-test-stdlib-${SWIFT_SDK_${SDK}_LIB_SUBDIR}")
237+
foreach(test_subset ${TEST_SUBSETS})
238+
set(want_this_combination TRUE)
239+
if(test_subset STREQUAL "only_long")
240+
# Long tests are only run in 'optimize_none' mode.
241+
if(NOT test_mode STREQUAL "optimize_none")
242+
set(want_this_combination FALSE)
243+
endif()
187244

188-
if(SWIFT_BUILD_STDLIB AND SWIFT_INCLUDE_TESTS)
189-
list(APPEND test_dependencies
190-
"swift-reflection-test${VARIANT_SUFFIX}")
191-
endif()
245+
list(APPEND LIT_ARGS "--param" "run_only_tests=long_tests")
246+
endif()
192247

193-
set(validation_test_dependencies
194-
"swiftStdlibCollectionUnittest-${SWIFT_SDK_${SDK}_LIB_SUBDIR}")
195-
196-
set(test_bin_dir "${CMAKE_CURRENT_BINARY_DIR}${VARIANT_SUFFIX}")
197-
set(validation_test_bin_dir
198-
"${CMAKE_CURRENT_BINARY_DIR}/../validation-test${VARIANT_SUFFIX}")
199-
200-
swift_configure_lit_site_cfg(
201-
"${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in"
202-
"${test_bin_dir}/lit.site.cfg"
203-
"test${VARIANT_SUFFIX}.lit.site.cfg")
204-
205-
swift_configure_lit_site_cfg(
206-
"${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in"
207-
"${test_bin_dir}/Unit/lit.site.cfg"
208-
"")
209-
210-
swift_configure_lit_site_cfg(
211-
"${CMAKE_CURRENT_SOURCE_DIR}/../validation-test/lit.site.cfg.in"
212-
"${validation_test_bin_dir}/lit.site.cfg"
213-
"validation-test${VARIANT_SUFFIX}.lit.site.cfg")
214-
set(profdata_merge_worker
215-
"${CMAKE_CURRENT_SOURCE_DIR}/../utils/profdata_merge/main.py")
216-
217-
if(SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "MERGED")
218-
set(command_profdata_merge_start
219-
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}"
220-
-l "${swift_test_results_dir}/profdata_merge.log"
221-
start
222-
-o "${swift_test_results_dir}")
223-
set(command_profdata_merge_stop
224-
COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" stop)
225-
else()
226-
set(command_profdata_merge_start)
227-
set(command_profdata_merge_stop)
228-
endif()
248+
set(directories)
249+
set(dependencies ${test_dependencies})
229250

230-
add_custom_target("check-swift${test_mode_target_suffix}${VARIANT_SUFFIX}"
231-
${command_upload_stdlib}
232-
${command_clean_test_results_dir}
233-
${command_profdata_merge_start}
234-
COMMAND ${lit_command} "${test_bin_dir}"
235-
${command_profdata_merge_stop}
236-
DEPENDS ${test_dependencies}
237-
COMMENT "Running Swift tests for ${VARIANT_TRIPLE}"
238-
${cmake_3_2_USES_TERMINAL})
239-
240-
add_custom_target("check-swift-validation${test_mode_target_suffix}${VARIANT_SUFFIX}"
241-
${command_upload_stdlib}
242-
${command_clean_test_results_dir}
243-
${command_profdata_merge_start}
244-
COMMAND ${lit_command} "${validation_test_bin_dir}"
245-
${command_profdata_merge_stop}
246-
DEPENDS ${test_dependencies} ${validation_test_dependencies}
247-
COMMENT "Running Swift validation tests for ${VARIANT_TRIPLE}"
248-
${cmake_3_2_USES_TERMINAL})
249-
250-
add_custom_target("check-swift-all${test_mode_target_suffix}${VARIANT_SUFFIX}"
251-
${command_upload_stdlib}
252-
${command_clean_test_results_dir}
253-
${command_profdata_merge_start}
254-
COMMAND ${lit_command} "${validation_test_bin_dir}" "${test_bin_dir}"
255-
${command_profdata_merge_stop}
256-
DEPENDS ${test_dependencies} ${validation_test_dependencies}
257-
COMMENT "Running all Swift tests for ${VARIANT_TRIPLE}"
258-
${cmake_3_2_USES_TERMINAL})
251+
if((test_subset STREQUAL "primary") OR
252+
(test_subset STREQUAL "only_long") OR
253+
(test_subset STREQUAL "all"))
254+
list(APPEND directories "${test_bin_dir}")
255+
endif()
256+
if((test_subset STREQUAL "validation") OR
257+
(test_subset STREQUAL "only_validation") OR
258+
(test_subset STREQUAL "only_long") OR
259+
(test_subset STREQUAL "all"))
260+
list(APPEND directories "${validation_test_bin_dir}")
261+
list(APPEND dependencies ${validation_test_dependencies})
262+
endif()
259263

264+
set(test_subset_target_suffix "-${test_subset}")
265+
if(test_subset STREQUAL "primary")
266+
set(test_subset_target_suffix "")
267+
endif()
260268

269+
if(want_this_combination)
270+
add_custom_target("check-swift${test_subset_target_suffix}${test_mode_target_suffix}${VARIANT_SUFFIX}"
271+
${command_upload_stdlib}
272+
${command_clean_test_results_dir}
273+
${command_profdata_merge_start}
274+
COMMAND ${PYTHON_EXECUTABLE} "${LIT}" ${LIT_ARGS} ${directories}
275+
${command_profdata_merge_stop}
276+
DEPENDS ${dependencies}
277+
COMMENT "Running ${test_subset} Swift tests for ${VARIANT_TRIPLE}"
278+
${cmake_3_2_USES_TERMINAL})
279+
endif()
280+
endforeach()
261281
endforeach()
262282
endforeach()
263283
endforeach()
264284

265285
# Add shortcuts for the default variant.
266-
foreach(TEST_MODE ${TEST_MODES})
267-
set(test_mode_target_suffix)
268-
if(TEST_MODE STREQUAL "optimize_none")
286+
foreach(test_mode ${TEST_MODES})
287+
foreach(test_subset ${TEST_SUBSETS})
269288
set(test_mode_target_suffix "")
270-
endif()
271-
if(TEST_MODE STREQUAL "optimize")
272-
set(test_mode_target_suffix "-optimize")
273-
endif()
274-
if(TEST_MODE STREQUAL "optimize_unchecked")
275-
set(test_mode_target_suffix "-optimize-unchecked")
276-
endif()
277-
if(TEST_MODE STREQUAL "executable")
278-
set(test_mode_target_suffix "-executable")
279-
endif()
280-
if(TEST_MODE STREQUAL "non_executable")
281-
set(test_mode_target_suffix "-non-executable")
282-
endif()
283-
284-
add_custom_target(check-swift${test_mode_target_suffix}
285-
DEPENDS "check-swift${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
286-
287-
add_custom_target(check-swift-validation${test_mode_target_suffix}
288-
DEPENDS "check-swift-validation${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
289-
290-
add_custom_target(check-swift-all${test_mode_target_suffix}
291-
DEPENDS "check-swift-all${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
292-
289+
if(NOT test_mode STREQUAL "optimize_none")
290+
set(test_mode_target_suffix "-${test_mode}")
291+
endif()
292+
set(test_subset_target_suffix "-${test_subset}")
293+
if(test_subset STREQUAL "primary")
294+
set(test_subset_target_suffix "")
295+
endif()
296+
297+
add_custom_target(check-swift${test_subset_target_suffix}${test_mode_target_suffix}
298+
DEPENDS "check-swift${test_subset_target_suffix}${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
299+
endforeach()
293300
endforeach()
294301

295302
endif()

‎test/lit.cfg

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,10 +458,10 @@ elif swift_test_mode == 'optimize_unchecked':
458458
config.available_features.add("swift_test_mode_optimize_unchecked_" + run_cpu)
459459
config.available_features.add("swift_test_mode_optimize_unchecked")
460460
swift_execution_tests_extra_flags = '-Ounchecked'
461-
elif swift_test_mode == 'executable':
461+
elif swift_test_mode == 'only_executable':
462462
# No extra flags needed.
463463
pass
464-
elif swift_test_mode == 'non_executable':
464+
elif swift_test_mode == 'only_non_executable':
465465
# No extra flags needed.
466466
pass
467467
else:
@@ -476,6 +476,9 @@ elif swift_run_only_tests == 'executable_tests':
476476
config.limit_to_features.add("executable_test")
477477
elif swift_run_only_tests == 'non_executable_tests':
478478
pass
479+
elif swift_run_only_tests == 'long_tests':
480+
config.available_features.add("long_test")
481+
config.limit_to_features.add("long_test")
479482
else:
480483
lit_config.fatal("Unknown test mode %r" % swift_run_only_tests)
481484

0 commit comments

Comments
 (0)