Skip to content

Commit 300cbab

Browse files
rxweihamishknight
rxwei
authored and
hamishknight
committed
Integrate experimental string processing modules and enable end-to-end regex.
- Checkout apple/swift-experimental-string-processing using a tag. - Build `_MatchingEngine` as part of libswift (`ExperimentalRegex`) using sources from the package. - Parse regex literals using the parser from `_MatchingEngine`. - Build both `_MatchingEngine` and `_StringProcessing` as part of core libs using sources from the package. - Use `Regex<DynamicCaptures>` as the default regex type until we finalize swiftlang/swift-experimental-string-processing#68.
1 parent 1647cdd commit 300cbab

File tree

22 files changed

+145
-90
lines changed

22 files changed

+145
-90
lines changed

‎cmake/modules/AddSwift.cmake

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -634,8 +634,13 @@ function(add_libswift_module module)
634634
""
635635
"DEPENDS"
636636
${ARGN})
637-
set(sources ${ALSM_UNPARSED_ARGUMENTS})
638-
list(TRANSFORM sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
637+
set(raw_sources ${ALSM_UNPARSED_ARGUMENTS})
638+
set(sources)
639+
foreach(raw_source ${raw_sources})
640+
get_filename_component(
641+
raw_source "${raw_source}" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
642+
list(APPEND sources "${raw_source}")
643+
endforeach()
639644
640645
set(target_name "LibSwift${module}")
641646

‎lib/Sema/CSGen.cpp

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,18 +1266,15 @@ namespace {
12661266
ctx.Id_Regex.str());
12671267
return Type();
12681268
}
1269-
auto substringType = ctx.getSubstringType();
12701269
auto dynCapturesType = ctx.getDynamicCapturesType();
12711270
if (!dynCapturesType) {
12721271
ctx.Diags.diagnose(E->getLoc(),
12731272
diag::string_processing_lib_missing,
12741273
"DynamicCaptures");
12751274
return Type();
12761275
}
1277-
// TODO: Replace `(Substring, DynamicCaptures)` with type inferred from
1278-
// the regex.
1279-
auto matchType = TupleType::get({substringType, dynCapturesType}, ctx);
1280-
return BoundGenericStructType::get(regexDecl, Type(), {matchType});
1276+
// TODO: Replace `DynamicCaptures` with type inferred from the regex.
1277+
return BoundGenericStructType::get(regexDecl, Type(), {dynCapturesType});
12811278
}
12821279

12831280
Type visitDeclRefExpr(DeclRefExpr *E) {

‎libswift/Sources/CMakeLists.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
# See http://swift.org/LICENSE.txt for license information
77
# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
88

9-
add_subdirectory(ExperimentalRegex)
9+
if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
10+
add_subdirectory(ExperimentalRegex)
11+
endif()
1012
add_subdirectory(SIL)
1113
add_subdirectory(Optimizer)

‎libswift/Sources/ExperimentalRegex/CMakeLists.txt

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,16 @@
66
# See http://swift.org/LICENSE.txt for license information
77
# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
88

9+
file(GLOB_RECURSE _LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES
10+
"${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
11+
set(LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES)
12+
foreach(source ${_LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES})
13+
file(TO_CMAKE_PATH "${source}" source)
14+
list(APPEND LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES ${source})
15+
endforeach()
16+
message(STATUS "Using Experimental String Processing library for libswift ExperimentalRegex (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
17+
918
add_libswift_module(ExperimentalRegex
10-
Regex.swift
11-
)
19+
"${LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES}"
20+
Regex.swift)
1221

‎libswift/Sources/ExperimentalRegex/Regex.swift

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ public func experimental_regex_strawperson(
55
) -> UnsafePointer<CChar>? {
66
guard let s = ptr else { return nil }
77

8-
func error(_ str: String) -> UnsafePointer<CChar>? {
8+
func makeError(_ str: String) -> UnsafePointer<CChar>? {
99
let count = str.utf8.count + 1
1010
return str.withCString {
1111
assert($0[count-1] == 0)
@@ -16,12 +16,15 @@ public func experimental_regex_strawperson(
1616
}
1717

1818
let str = String(cString: s)
19-
if str.starts(with: "*") || str.starts(with: "+") || str.starts(with: "?") {
20-
return error("cannot start regex with quantifier")
19+
do {
20+
let _ = try parse(str, .traditional)
21+
return nil
22+
} catch {
23+
return makeError(
24+
"cannot parse regular expression: \(String(describing: error))")
2125
}
22-
return nil
2326
}
2427

25-
public func registerParser() {
28+
public func registerRegexParser() {
2629
Parser_registerParseRegexStrawperson({ experimental_regex_strawperson($0) })
2730
}

‎libswift/Sources/Optimizer/CMakeLists.txt

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66
# See http://swift.org/LICENSE.txt for license information
77
# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
88

9-
add_libswift_module(Optimizer
10-
DEPENDS SIL ExperimentalRegex)
9+
set(dependencies)
10+
list(APPEND dependencies SIL)
11+
if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
12+
list(APPEND dependencies ExperimentalRegex)
13+
endif()
14+
15+
add_libswift_module(Optimizer DEPENDS ${dependencies})
1116

1217
add_subdirectory(Analysis)
1318
add_subdirectory(InstructionPasses)

‎libswift/Sources/Optimizer/PassManager/PassRegistration.swift

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,19 @@
1212

1313
import SIL
1414
import OptimizerBridging
15+
16+
#if canImport(ExperimentalRegex)
1517
import ExperimentalRegex
18+
#endif
1619

1720
@_cdecl("initializeLibSwift")
1821
public func initializeLibSwift() {
1922
registerSILClasses()
2023
registerSwiftPasses()
21-
registerParser()
24+
25+
#if canImport(ExperimentalRegex)
26+
registerRegexParser()
27+
#endif
2228
}
2329

2430
private func registerPass(

‎stdlib/public/MatchingEngine/CMakeLists.txt

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,17 @@
1313
set(swift_matching_engine_link_libraries
1414
swiftCore)
1515

16+
file(GLOB_RECURSE _MATCHING_ENGINE_SOURCES
17+
"${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
18+
set(MATCHING_ENGINE_SOURCES)
19+
foreach(source ${_MATCHING_ENGINE_SOURCES})
20+
file(TO_CMAKE_PATH "${source}" source)
21+
list(APPEND MATCHING_ENGINE_SOURCES ${source})
22+
endforeach()
23+
message(STATUS "Using Experimental String Processing library for _MatchingEngine (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
1624

1725
add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
18-
MatchingEngine.swift
26+
"${MATCHING_ENGINE_SOURCES}"
1927

2028
SWIFT_MODULE_DEPENDS_LINUX Glibc
2129
SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -30,8 +38,6 @@ add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES
3038
-Dswift_MatchingEngine_EXPORTS
3139
SWIFT_COMPILE_FLAGS
3240
${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
33-
-parse-stdlib
34-
-Xfrontend -enable-experimental-string-processing
3541
LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
3642

3743
INSTALL_IN_COMPONENT stdlib

‎stdlib/public/MatchingEngine/MatchingEngine.swift

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎stdlib/public/StringProcessing/CMakeLists.txt

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,17 @@ set(swift_string_processing_link_libraries
1414
swiftCore
1515
swift_MatchingEngine)
1616

17+
file(GLOB_RECURSE _STRING_PROCESSING_SOURCES
18+
"${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_StringProcessing/*.swift")
19+
set(STRING_PROCESSING_SOURCES)
20+
foreach(source ${_STRING_PROCESSING_SOURCES})
21+
file(TO_CMAKE_PATH "${source}" source)
22+
list(APPEND STRING_PROCESSING_SOURCES ${source})
23+
endforeach()
24+
message(STATUS "Using Experimental String Processing library for _StringProcessing (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
1725

1826
add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
19-
Regex.swift
27+
"${STRING_PROCESSING_SOURCES}"
2028

2129
SWIFT_MODULE_DEPENDS_LINUX Glibc
2230
SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -31,8 +39,6 @@ add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYP
3139
-Dswift_StringProcessing_EXPORTS
3240
SWIFT_COMPILE_FLAGS
3341
${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
34-
-parse-stdlib
35-
-Xfrontend -enable-experimental-string-processing
3642
LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
3743

3844
SWIFT_MODULE_DEPENDS _MatchingEngine

‎stdlib/public/StringProcessing/Regex.swift

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎test/StringProcessing/Parse/regex.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
// RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
22
// REQUIRES: libswift
33

4-
var s = 'abc'
4+
_ = 'abc'
55

6-
var s1 = ('*', '+', '?')
7-
// expected-error@-1 3{{cannot start regex with quantifier}}
6+
_ = ('[*', '+]', '.]')
7+
// expected-error@-1 {{cannot parse regular expression}}
88

9-
var s2 = '\w+'
10-
var s3 = '\'\\'
9+
_ = '\w+'
10+
_ = '\'\\'

‎test/StringProcessing/Runtime/regex_basic.swift

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// RUN: %target-run-simple-swift(-Xfrontend -enable-experimental-string-processing)
2+
3+
// REQUIRES: libswift,string_processing,executable_test
4+
5+
import StdlibUnittest
6+
7+
var RegexBasicTests = TestSuite("RegexBasic")
8+
9+
extension String {
10+
func expectMatch<T>(
11+
_ regex: Regex<T>,
12+
file: String = #file,
13+
line: UInt = #line
14+
) -> RegexMatch<T> {
15+
guard let result = match(regex) else {
16+
expectUnreachable("Failed match", file: file, line: line)
17+
fatalError()
18+
}
19+
return result
20+
}
21+
}
22+
23+
RegexBasicTests.test("Basic") {
24+
let input = "aabccd"
25+
26+
let match1 = input.expectMatch('aabcc.')
27+
expectEqual("aabccd", input[match1.range])
28+
expectEqual(.empty, match1.captures)
29+
30+
let match2 = input.expectMatch('a*b.+.')
31+
expectEqual("aabccd", input[match2.range])
32+
expectEqual(.empty, match2.captures)
33+
}
34+
35+
RegexBasicTests.test("Captures") {
36+
let input = """
37+
A6F0..A6F1 ; Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM \
38+
COMBINING MARK TUKWENTIS
39+
"""
40+
let regex = '([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+;\s+(\w+).*'
41+
let match1 = input.expectMatch(regex)
42+
expectEqual(input[...], input[match1.range])
43+
expectEqual(
44+
.tuple([
45+
.substring("A6F0"),
46+
.optional(.substring("A6F1")),
47+
.substring("Extend")
48+
]),
49+
match1.captures)
50+
}
51+
52+
runAllTests()

‎test/StringProcessing/SILGen/regex_literal_silgen.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ var s = 'abc'
55
// CHECK: [[REGEX_STR_LITERAL:%[0-9]+]] = string_literal utf8 "abc"
66
// CHECK: [[STRING_INIT:%[0-9]+]] = function_ref @$sSS21_builtinStringLiteral17utf8CodeUnitCount7isASCIISSBp_BwBi1_tcfC : $@convention(method) (Builtin.RawPointer, Builtin.Word, Builtin.Int1, @thin String.Type) -> @owned String
77
// CHECK: [[REGEX_STR:%[0-9]+]] = apply [[STRING_INIT]]([[REGEX_STR_LITERAL]]
8-
// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
9-
// CHECK: apply [[REGEX_INIT]]<(Substring, DynamicCaptures)>([[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
8+
// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>
9+
// CHECK: apply [[REGEX_INIT]]<DynamicCaptures>({{%.+}}, [[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>

‎test/StringProcessing/Sema/regex_literal_type_inference.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33

44
let r0 = '.' // okay
55

6-
let r1: Regex<(Substring, DynamicCaptures)> = '.(.)' // okay
6+
let r1: Regex<DynamicCaptures> = '.(.)' // okay
77

8-
// expected-error @+2 {{cannot assign value of type 'Regex<(Substring, DynamicCaptures)>' to type 'Regex<Substring>'}}
9-
// expected-note @+1 {{arguments to generic parameter 'Match' ('(Substring, DynamicCaptures)' and 'Substring') are expected to be equal}}
8+
// expected-error @+2 {{cannot assign value of type 'Regex<DynamicCaptures>' to type 'Regex<Substring>'}}
9+
// expected-note @+1 {{arguments to generic parameter 'Capture' ('DynamicCaptures' and 'Substring') are expected to be equal}}
1010
let r2: Regex<Substring> = '.(.)'
1111

1212
func takesRegex<Match>(_: Regex<Match>) {}

‎test/StringProcessing/lit.local.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
if 'string_processing' not in config.available_features:
2-
config.unsupported = False
2+
config.unsupported = True

‎test/StringProcessing/string_processing_dummy.swift

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎utils/build-script-impl

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ KNOWN_SETTINGS=(
225225
swift-stdlib-experimental-hermetic-seal-at-link "0" "whether stdlib should be built with -experimental-hermetic-seal-at-link"
226226
swift-disable-dead-stripping "0" "turns off Darwin-specific dead stripping for Swift host tools"
227227
common-swift-flags "" "Flags used for Swift targets other than the stdlib, like the corelibs"
228+
swift-enable-experimental-string-processing "1" "whether to build experimental string processing feature"
228229

229230
## FREESTANDING Stdlib Options
230231
swift-freestanding-flavor "" "when building the FREESTANDING stdlib, which build style to use (options: apple, linux)"
@@ -1252,6 +1253,7 @@ LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
12521253
LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
12531254
LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
12541255
LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/libcxx"
1256+
EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${WORKSPACE}/swift-experimental-string-processing"
12551257

12561258
# We cannot currently apply the normal rules of skipping here for LLVM. Even if
12571259
# we are skipping building LLVM, we still need to at least build a few tools
@@ -2196,6 +2198,14 @@ for host in "${ALL_HOSTS[@]}"; do
21962198
)
21972199
fi
21982200

2201+
if [[ $(true_false "${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}") == "TRUE" ]] ; then
2202+
cmake_options=(
2203+
"${cmake_options[@]}"
2204+
-DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL=TRUE
2205+
-DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}"
2206+
)
2207+
fi
2208+
21992209
build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}")
22002210
if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then
22012211
native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)"

‎utils/build-windows.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ cmake^
272272
-DSWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED=YES^
273273
-DSWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING=YES^
274274
-DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING=YES^
275+
-DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR=%source_root%\swift-experimental-string-processing^
275276
-DSWIFT_INSTALL_COMPONENTS="autolink-driver;compiler;clang-resource-dir-symlink;stdlib;sdk-overlay;editor-integration;tools;testsuite-tools;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers"^
276277
-DSWIFT_PARALLEL_LINK_JOBS=8^
277278
-DPYTHON_EXECUTABLE:PATH=%PYTHON_HOME%\python.exe^

‎utils/swift_build_support/swift_build_support/products/swift.py

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,6 @@ def __init__(self, args, toolchain, source_dir, build_dir):
5959
# Add static vprintf flag
6060
self.cmake_options.extend(self._enable_stdlib_static_vprintf)
6161

62-
# Add experimental string processing flag.
63-
self.cmake_options.extend(self._enable_experimental_string_processing)
64-
6562
# Add freestanding related flags.
6663
self.cmake_options.extend(self._freestanding_is_darwin)
6764

@@ -171,11 +168,6 @@ def _enable_stdlib_static_vprintf(self):
171168
return [('SWIFT_STDLIB_STATIC_PRINT',
172169
self.args.build_swift_stdlib_static_print)]
173170

174-
@property
175-
def _enable_experimental_string_processing(self):
176-
return [('SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL',
177-
self.args.enable_experimental_string_processing)]
178-
179171
@property
180172
def _freestanding_is_darwin(self):
181173
return [('SWIFT_FREESTANDING_IS_DARWIN:BOOL',

0 commit comments

Comments
 (0)