Integrate experimental string processing modules and enable end-to-end regex.

cmake/modules/AddSwift.cmake

@@ -634,8 +634,13 @@ function(add_libswift_module module)
                         ""
                         "DEPENDS"
                         ${ARGN})
-  set(sources ${ALSM_UNPARSED_ARGUMENTS})
-  list(TRANSFORM sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
+  set(raw_sources ${ALSM_UNPARSED_ARGUMENTS})
+  set(sources)
+  foreach(raw_source ${raw_sources})
+    get_filename_component(
+      raw_source "${raw_source}" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+    list(APPEND sources "${raw_source}")
+  endforeach()
 
   set(target_name "LibSwift${module}")
 

lib/Sema/CSGen.cpp

@@ -1266,18 +1266,15 @@ namespace {
                            ctx.Id_Regex.str());
         return Type();
       }
-      auto substringType = ctx.getSubstringType();
       auto dynCapturesType = ctx.getDynamicCapturesType();
       if (!dynCapturesType) {
         ctx.Diags.diagnose(E->getLoc(),
                            diag::string_processing_lib_missing,
                            "DynamicCaptures");
         return Type();
       }
-      // TODO: Replace `(Substring, DynamicCaptures)` with type inferred from
-      // the regex.
-      auto matchType = TupleType::get({substringType, dynCapturesType}, ctx);
-      return BoundGenericStructType::get(regexDecl, Type(), {matchType});
+      // TODO: Replace `DynamicCaptures` with type inferred from the regex.
+      return BoundGenericStructType::get(regexDecl, Type(), {dynCapturesType});
     }
 
     Type visitDeclRefExpr(DeclRefExpr *E) {

libswift/Sources/CMakeLists.txt

@@ -6,6 +6,8 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-add_subdirectory(ExperimentalRegex)
+if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+  add_subdirectory(ExperimentalRegex)
+endif()
 add_subdirectory(SIL)
 add_subdirectory(Optimizer)

libswift/Sources/ExperimentalRegex/CMakeLists.txt

@@ -6,7 +6,16 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
+file(GLOB_RECURSE _LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES
+  "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
+set(LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES)
+foreach(source ${_LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES})
+  file(TO_CMAKE_PATH "${source}" source)
+  list(APPEND LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for libswift ExperimentalRegex (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
+
 add_libswift_module(ExperimentalRegex
-  Regex.swift
-)
+  "${LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES}"
+  Regex.swift)
 

libswift/Sources/ExperimentalRegex/Regex.swift

@@ -5,7 +5,7 @@ public func experimental_regex_strawperson(
 ) -> UnsafePointer<CChar>? {
   guard let s = ptr else { return nil }
 
-  func error(_ str: String) -> UnsafePointer<CChar>? {
+  func makeError(_ str: String) -> UnsafePointer<CChar>? {
     let count = str.utf8.count + 1
     return str.withCString {
       assert($0[count-1] == 0)
@@ -16,12 +16,15 @@ public func experimental_regex_strawperson(
   }
 
   let str = String(cString: s)
-  if str.starts(with: "*") || str.starts(with: "+") || str.starts(with: "?") {
-    return error("cannot start regex with quantifier")
+  do {
+    let _ = try parse(str, .traditional)
+    return nil
+  } catch {
+    return makeError(
+      "cannot parse regular expression: \(String(describing: error))")
   }
-  return nil
 }
 
-public func registerParser() {
+public func registerRegexParser() {
   Parser_registerParseRegexStrawperson({ experimental_regex_strawperson($0) })
 }

libswift/Sources/Optimizer/CMakeLists.txt

@@ -6,8 +6,13 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-add_libswift_module(Optimizer
-    DEPENDS SIL ExperimentalRegex)
+set(dependencies)
+list(APPEND dependencies SIL)
+if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+  list(APPEND dependencies ExperimentalRegex)
+endif()
+
+add_libswift_module(Optimizer DEPENDS ${dependencies})
 
 add_subdirectory(Analysis)
 add_subdirectory(InstructionPasses)

libswift/Sources/Optimizer/PassManager/PassRegistration.swift

@@ -12,13 +12,19 @@
 
 import SIL
 import OptimizerBridging
+
+#if canImport(ExperimentalRegex)
 import ExperimentalRegex
+#endif
 
 @_cdecl("initializeLibSwift")
 public func initializeLibSwift() {
   registerSILClasses()
   registerSwiftPasses()
-  registerParser()
+
+  #if canImport(ExperimentalRegex)
+  registerRegexParser()
+  #endif
 }
 
 private func registerPass(

stdlib/public/MatchingEngine/CMakeLists.txt

@@ -13,9 +13,17 @@
 set(swift_matching_engine_link_libraries
   swiftCore)
 
+file(GLOB_RECURSE _MATCHING_ENGINE_SOURCES
+  "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
+set(MATCHING_ENGINE_SOURCES)
+foreach(source ${_MATCHING_ENGINE_SOURCES})
+  file(TO_CMAKE_PATH "${source}" source)
+  list(APPEND MATCHING_ENGINE_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for _MatchingEngine (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
 
 add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
-  MatchingEngine.swift
+  "${MATCHING_ENGINE_SOURCES}"
 
   SWIFT_MODULE_DEPENDS_LINUX Glibc
   SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -30,8 +38,6 @@ add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES
     -Dswift_MatchingEngine_EXPORTS
   SWIFT_COMPILE_FLAGS
     ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
-    -parse-stdlib
-    -Xfrontend -enable-experimental-string-processing
   LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
 
   INSTALL_IN_COMPONENT stdlib

stdlib/public/StringProcessing/CMakeLists.txt

@@ -14,9 +14,17 @@ set(swift_string_processing_link_libraries
   swiftCore
   swift_MatchingEngine)
 
+file(GLOB_RECURSE _STRING_PROCESSING_SOURCES
+  "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_StringProcessing/*.swift")
+set(STRING_PROCESSING_SOURCES)
+foreach(source ${_STRING_PROCESSING_SOURCES})
+  file(TO_CMAKE_PATH "${source}" source)
+  list(APPEND STRING_PROCESSING_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for _StringProcessing (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
 
 add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
-  Regex.swift
+  "${STRING_PROCESSING_SOURCES}"
 
   SWIFT_MODULE_DEPENDS_LINUX Glibc
   SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -31,8 +39,6 @@ add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYP
     -Dswift_StringProcessing_EXPORTS
   SWIFT_COMPILE_FLAGS
     ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
-    -parse-stdlib
-    -Xfrontend -enable-experimental-string-processing
   LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
 
   SWIFT_MODULE_DEPENDS _MatchingEngine

test/StringProcessing/Parse/regex.swift

@@ -1,10 +1,10 @@
 // RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
 // REQUIRES: libswift
 
-var s = 'abc'
+_ = 'abc'
 
-var s1 = ('*', '+', '?')
-// expected-error@-1 3{{cannot start regex with quantifier}}
+_ = ('[*', '+]', '.]')
+// expected-error@-1 {{cannot parse regular expression}}
 
-var s2 = '\w+'
-var s3 = '\'\\'
+_ = '\w+'
+_ = '\'\\'

test/StringProcessing/Runtime/regex_basic.swift

@@ -0,0 +1,52 @@
+// RUN: %target-run-simple-swift(-Xfrontend -enable-experimental-string-processing)
+
+// REQUIRES: libswift,string_processing,executable_test
+
+import StdlibUnittest
+
+var RegexBasicTests = TestSuite("RegexBasic")
+
+extension String {
+  func expectMatch<T>(
+    _ regex: Regex<T>,
+    file: String = #file,
+    line: UInt = #line
+  ) -> RegexMatch<T> {
+    guard let result = match(regex) else {
+      expectUnreachable("Failed match", file: file, line: line)
+      fatalError()
+    }
+    return result
+  }
+}
+
+RegexBasicTests.test("Basic") {
+  let input = "aabccd"
+
+  let match1 = input.expectMatch('aabcc.')
+  expectEqual("aabccd", input[match1.range])
+  expectEqual(.empty, match1.captures)
+
+  let match2 = input.expectMatch('a*b.+.')
+  expectEqual("aabccd", input[match2.range])
+  expectEqual(.empty, match2.captures)
+}
+
+RegexBasicTests.test("Captures") {
+  let input = """
+    A6F0..A6F1    ; Extend # Mn   [2] BAMUM COMBINING MARK KOQNDON..BAMUM \
+    COMBINING MARK TUKWENTIS
+    """
+  let regex = '([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+;\s+(\w+).*'
+  let match1 = input.expectMatch(regex)
+  expectEqual(input[...], input[match1.range])
+  expectEqual(
+    .tuple([
+      .substring("A6F0"),
+      .optional(.substring("A6F1")),
+      .substring("Extend")
+    ]),
+    match1.captures)
+}
+
+runAllTests()

test/StringProcessing/SILGen/regex_literal_silgen.swift

@@ -5,5 +5,5 @@ var s = 'abc'
 // CHECK: [[REGEX_STR_LITERAL:%[0-9]+]] = string_literal utf8 "abc"
 // CHECK: [[STRING_INIT:%[0-9]+]] = function_ref @$sSS21_builtinStringLiteral17utf8CodeUnitCount7isASCIISSBp_BwBi1_tcfC : $@convention(method) (Builtin.RawPointer, Builtin.Word, Builtin.Int1, @thin String.Type) -> @owned String
 // CHECK: [[REGEX_STR:%[0-9]+]] = apply [[STRING_INIT]]([[REGEX_STR_LITERAL]]
-// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
-// CHECK: apply [[REGEX_INIT]]<(Substring, DynamicCaptures)>([[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
+// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>
+// CHECK: apply [[REGEX_INIT]]<DynamicCaptures>({{%.+}}, [[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>

test/StringProcessing/Sema/regex_literal_type_inference.swift

@@ -3,10 +3,10 @@
 
 let r0 = '.' // okay
 
-let r1: Regex<(Substring, DynamicCaptures)> = '.(.)' // okay
+let r1: Regex<DynamicCaptures> = '.(.)' // okay
 
-// expected-error @+2 {{cannot assign value of type 'Regex<(Substring, DynamicCaptures)>' to type 'Regex<Substring>'}}
-// expected-note @+1 {{arguments to generic parameter 'Match' ('(Substring, DynamicCaptures)' and 'Substring') are expected to be equal}}
+// expected-error @+2 {{cannot assign value of type 'Regex<DynamicCaptures>' to type 'Regex<Substring>'}}
+// expected-note @+1 {{arguments to generic parameter 'Capture' ('DynamicCaptures' and 'Substring') are expected to be equal}}
 let r2: Regex<Substring> = '.(.)'
 
 func takesRegex<Match>(_: Regex<Match>) {}

test/StringProcessing/lit.local.cfg

@@ -1,2 +1,2 @@
 if 'string_processing' not in config.available_features:
-    config.unsupported = False
+    config.unsupported = True

utils/build-script-impl

@@ -225,6 +225,7 @@ KNOWN_SETTINGS=(
     swift-stdlib-experimental-hermetic-seal-at-link     "0"         "whether stdlib should be built with -experimental-hermetic-seal-at-link"
     swift-disable-dead-stripping                  "0"               "turns off Darwin-specific dead stripping for Swift host tools"
     common-swift-flags                            ""                "Flags used for Swift targets other than the stdlib, like the corelibs"
+    swift-enable-experimental-string-processing   "1"               "whether to build experimental string processing feature"
 
     ## FREESTANDING Stdlib Options
     swift-freestanding-flavor                     ""                "when building the FREESTANDING stdlib, which build style to use (options: apple, linux)"
@@ -1252,6 +1253,7 @@ LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
 LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/libcxx"
+EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${WORKSPACE}/swift-experimental-string-processing"
 
 # We cannot currently apply the normal rules of skipping here for LLVM. Even if
 # we are skipping building LLVM, we still need to at least build a few tools
@@ -2196,6 +2198,14 @@ for host in "${ALL_HOSTS[@]}"; do
                     )
                 fi
 
+                if [[ $(true_false "${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}") == "TRUE" ]] ; then
+                    cmake_options=(
+                        "${cmake_options[@]}"
+                        -DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL=TRUE
+                        -DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}"
+                    )
+                fi
+
                 build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}")
                 if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then
                     native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)"

utils/build-windows.bat

@@ -272,6 +272,7 @@ cmake^
     -DSWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED=YES^
     -DSWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING=YES^
     -DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING=YES^
+    -DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR=%source_root%\swift-experimental-string-processing^
     -DSWIFT_INSTALL_COMPONENTS="autolink-driver;compiler;clang-resource-dir-symlink;stdlib;sdk-overlay;editor-integration;tools;testsuite-tools;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers"^
     -DSWIFT_PARALLEL_LINK_JOBS=8^
     -DPYTHON_EXECUTABLE:PATH=%PYTHON_HOME%\python.exe^

utils/swift_build_support/swift_build_support/products/swift.py

@@ -59,9 +59,6 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         # Add static vprintf flag
         self.cmake_options.extend(self._enable_stdlib_static_vprintf)
 
-        # Add experimental string processing flag.
-        self.cmake_options.extend(self._enable_experimental_string_processing)
-
         # Add freestanding related flags.
         self.cmake_options.extend(self._freestanding_is_darwin)
 
@@ -171,11 +168,6 @@ def _enable_stdlib_static_vprintf(self):
         return [('SWIFT_STDLIB_STATIC_PRINT',
                  self.args.build_swift_stdlib_static_print)]
 
-    @property
-    def _enable_experimental_string_processing(self):
-        return [('SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL',
-                 self.args.enable_experimental_string_processing)]
-
     @property
     def _freestanding_is_darwin(self):
         return [('SWIFT_FREESTANDING_IS_DARWIN:BOOL',