Regex literal runtime plumbing.

- Frontend: Implicitly import `_StringProcessing` when frontend flag `-enable-experimental-string-processing` is set.
- Type checker: Set a regex literal expression's type as `_StringProcessing.Regex<(Substring, DynamicCaptures)>`. `(Substring, DynamicCaptures)` is a temporary `Match` type that will help get us to an end-to-end working system. This will be replaced by actual type inference based a regex's pattern in a follow-up patch (soon).
- SILGen: Lower a regex literal expression to a call to `_StringProcessing.Regex.init(_regexString:)`.
- String processing runtime: Add `Regex`, `DynamicCaptures` (matching actual APIs in apple/swift-experimental-string-processing), and `Regex(_regexString:)`.

Upcoming:
- Build `_MatchingEngine` and `_StringProcessing` modules with sources from apple/swift-experimental-string-processing.
- Replace `DynamicCaptures` with inferred capture types.

Author: rxwei

include/swift/AST/ASTContext.h

@@ -620,6 +620,9 @@ class ASTContext final {
   ConcreteDeclRef getBuiltinInitDecl(NominalTypeDecl *decl,
                                      KnownProtocolKind builtinProtocol,
                 llvm::function_ref<DeclName (ASTContext &ctx)> initName) const;
+
+  /// Retrieve _StringProcessing.Regex.init(_regexString: String).
+  ConcreteDeclRef getRegexInitDecl(Type regexType) const;
 
   /// Retrieve the declaration of Swift.<(Int, Int) -> Bool.
   FuncDecl *getLessThanIntDecl() const;

include/swift/AST/DiagnosticsFrontend.def

@@ -145,6 +145,8 @@ WARNING(emit_reference_dependencies_without_primary_file,none,
 
 WARNING(warn_implicit_concurrency_import_failed,none,
         "unable to perform implicit import of \"_Concurrency\" module: no such module found", ())
+WARNING(warn_implicit_string_processing_import_failed,none,
+        "unable to perform implicit import of \"_StringProcessing\" module: no such module found", ())
 
 ERROR(error_module_name_required,none, "-module-name is required", ())
 ERROR(error_bad_module_name,none,

include/swift/AST/DiagnosticsSema.def

@@ -3668,9 +3668,6 @@ ERROR(builtin_string_literal_broken_proto,none,
 ERROR(string_literal_broken_proto,none,
       "protocol 'ExpressibleByStringLiteral' is broken", ())
 
-ERROR(regex_decl_broken,none,
-      "cannot find 'Regex' type in scope", ())
-
 // Array literals
 ERROR(should_use_dictionary_literal,none,
       "dictionary of type %0 cannot be %select{used|initialized}1 "
@@ -4722,6 +4719,14 @@ ERROR(async_unavailable_decl,none,
       "%0 %1 is unavailable from asynchronous contexts%select{|; %3}2",
       (DescriptiveDeclKind, DeclBaseName, bool, StringRef))
 
+//------------------------------------------------------------------------------
+// MARK: String Processing
+//------------------------------------------------------------------------------
+
+ERROR(string_processing_lib_missing,none,
+      "missing '%0' declaration, probably because the '_StringProcessing' "
+      "module was not imported properly", (StringRef))
+
 //------------------------------------------------------------------------------
 // MARK: Type Check Types
 //------------------------------------------------------------------------------

include/swift/AST/Expr.h

@@ -966,26 +966,18 @@ class InterpolatedStringLiteralExpr : public LiteralExpr {
 class RegexLiteralExpr : public LiteralExpr {
   SourceLoc Loc;
   StringRef RegexText;
-  Expr *SemanticExpr;
 
-  RegexLiteralExpr(SourceLoc loc, StringRef regexText, Expr *semanticExpr,
-                   bool isImplicit)
+  RegexLiteralExpr(SourceLoc loc, StringRef regexText, bool isImplicit)
       : LiteralExpr(ExprKind::RegexLiteral, isImplicit), Loc(loc),
-        RegexText(regexText), SemanticExpr(semanticExpr) {}
+        RegexText(regexText) {}
 
 public:
   static RegexLiteralExpr *createParsed(ASTContext &ctx, SourceLoc loc,
-                                        StringRef regexText,
-                                        Expr *semanticExpr);
+                                        StringRef regexText);
 
   /// Retrieve the raw regex text.
   StringRef getRegexText() const { return RegexText; }
 
-  /// Retrieve the semantic expression that the regex will be type-checked and
-  /// emitted as.
-  Expr *getSemanticExpr() const { return SemanticExpr; }
-  void setSemanticExpr(Expr *expr) { SemanticExpr = expr; }
-
   SourceRange getSourceRange() const { return Loc; }
 
   static bool classof(const Expr *E) {

include/swift/AST/KnownIdentifiers.def

@@ -250,9 +250,10 @@ IDENTIFIER(pullback)
 IDENTIFIER(TangentVector)
 IDENTIFIER(zero)
 
-// Regex literals
+// String processing
 IDENTIFIER(Regex)
-IDENTIFIER(_regexString)
+IDENTIFIER_(regexString)
+IDENTIFIER_(StringProcessing)
 
 // Distributed actors
 IDENTIFIER(transport)

include/swift/AST/KnownSDKTypes.def

@@ -48,4 +48,8 @@ KNOWN_SDK_TYPE_DECL(Distributed, DistributedActor, ProtocolDecl, 0)
 KNOWN_SDK_TYPE_DECL(Distributed, ActorIdentity, ProtocolDecl, 0)
 KNOWN_SDK_TYPE_DECL(Distributed, AnyActorIdentity, StructDecl, 0)
 
+// String processing
+KNOWN_SDK_TYPE_DECL(StringProcessing, Regex, StructDecl, 1)
+KNOWN_SDK_TYPE_DECL(StringProcessing, DynamicCaptures, EnumDecl, 0)
+
 #undef KNOWN_SDK_TYPE_DECL

include/swift/Frontend/Frontend.h

@@ -354,6 +354,10 @@ class CompilerInvocation {
   /// imported.
   bool shouldImportSwiftConcurrency() const;
 
+  /// Whether the Swift String Processing support library should be implicitly
+  /// imported.
+  bool shouldImportSwiftStringProcessing() const;
+
   /// Performs input setup common to these tools:
   /// sil-opt, sil-func-extractor, sil-llvm-gen, and sil-nm.
   /// Return value includes the buffer so caller can keep it alive.
@@ -533,6 +537,14 @@ class CompilerInstance {
   /// i.e. if it can be found.
   bool canImportSwiftConcurrency() const;
 
+  /// Verify that if an implicit import of the `StringProcessing` module if
+  /// expected, it can actually be imported. Emit a warning, otherwise.
+  void verifyImplicitStringProcessingImport();
+
+  /// Whether the Swift String Processing support library can be imported
+  /// i.e. if it can be found.
+  bool canImportSwiftStringProcessing() const;
+
   /// Gets the SourceFile which is the primary input for this CompilerInstance.
   /// \returns the primary SourceFile, or nullptr if there is no primary input;
   /// if there are _multiple_ primary inputs, fails with an assertion.

include/swift/Strings.h

@@ -26,6 +26,8 @@ constexpr static const StringLiteral SWIFT_ONONE_SUPPORT = "SwiftOnoneSupport";
 constexpr static const StringLiteral SWIFT_CONCURRENCY_NAME = "_Concurrency";
 /// The name of the Distributed module, which supports that extension.
 constexpr static const StringLiteral SWIFT_DISTRIBUTED_NAME = "_Distributed";
+/// The name of the StringProcessing module, which supports that extension.
+constexpr static const StringLiteral SWIFT_STRING_PROCESSING_NAME = "_StringProcessing";
 /// The name of the SwiftShims module, which contains private stdlib decls.
 constexpr static const StringLiteral SWIFT_SHIMS_NAME = "SwiftShims";
 /// The name of the Builtin module, which contains Builtin functions.

lib/AST/ASTContext.cpp

@@ -1216,8 +1216,23 @@ ASTContext::getBuiltinInitDecl(NominalTypeDecl *decl,
   return witness;
 }
 
+ConcreteDeclRef ASTContext::getRegexInitDecl(Type regexType) const {
+  auto *spModule = getLoadedModule(Id_StringProcessing);
+  DeclName name(*const_cast<ASTContext *>(this),
+                DeclBaseName::createConstructor(),
+                {Id_regexString});
+  SmallVector<ValueDecl *, 1> results;
+  spModule->lookupQualified(getRegexType(), DeclNameRef(name),
+                            NL_IncludeUsableFromInline, results);
+  assert(results.size() == 1);
+  auto *foundDecl = cast<ConstructorDecl>(results[0]);
+  auto subs = regexType->getMemberSubstitutionMap(spModule, foundDecl);
+  return ConcreteDeclRef(foundDecl, subs);
+}
+
 static 
-FuncDecl *getBinaryComparisonOperatorIntDecl(const ASTContext &C, StringRef op, FuncDecl *&cached) {
+FuncDecl *getBinaryComparisonOperatorIntDecl(const ASTContext &C, StringRef op,
+                                             FuncDecl *&cached) {
   if (cached)
     return cached;
 

lib/AST/ASTDumper.cpp

@@ -1909,7 +1909,10 @@ class PrintExpr : public ExprVisitor<PrintExpr> {
   }
   void visitRegexLiteralExpr(RegexLiteralExpr *E) {
     printCommon(E, "regex_literal_expr");
-    printRec(E->getSemanticExpr());
+    PrintWithColorRAII(OS, LiteralValueColor)
+        << " text=" << QuotedString(E->getRegexText())
+        << " initializer=";
+    E->getInitializer().dump(PrintWithColorRAII(OS, LiteralValueColor).getOS());
     PrintWithColorRAII(OS, ParenthesisColor) << ')';
   }
 

lib/AST/ASTWalker.cpp

@@ -1153,11 +1153,6 @@ class Traversal : public ASTVisitor<Traversal, Expr*, Stmt*,
   }
 
   Expr *visitRegexLiteralExpr(RegexLiteralExpr *E) {
-    if (auto *newExpr = doIt(E->getSemanticExpr())) {
-      E->setSemanticExpr(newExpr);
-    } else {
-      return nullptr;
-    }
     return E;
   }
 

lib/AST/Expr.cpp

@@ -2212,9 +2212,8 @@ SourceLoc TapExpr::getEndLoc() const {
 
 RegexLiteralExpr *
 RegexLiteralExpr::createParsed(ASTContext &ctx, SourceLoc loc,
-                               StringRef regexText, Expr *semanticExpr) {
-  return new (ctx) RegexLiteralExpr(loc, regexText, semanticExpr,
-                                    /*implicit*/ false);
+                               StringRef regexText) {
+  return new (ctx) RegexLiteralExpr(loc, regexText, /*implicit*/ false);
 }
 
 void swift::simple_display(llvm::raw_ostream &out, const ClosureExpr *CE) {

lib/Frontend/Frontend.cpp

@@ -793,6 +793,10 @@ bool CompilerInvocation::shouldImportSwiftConcurrency() const {
         FrontendOptions::ParseInputMode::SwiftModuleInterface;
 }
 
+bool CompilerInvocation::shouldImportSwiftStringProcessing() const {
+  return getLangOptions().EnableExperimentalStringProcessing;
+}
+
 /// Implicitly import the SwiftOnoneSupport module in non-optimized
 /// builds. This allows for use of popular specialized functions
 /// from the standard library, which makes the non-optimized builds
@@ -833,6 +837,20 @@ bool CompilerInstance::canImportSwiftConcurrency() const {
       {getASTContext().getIdentifier(SWIFT_CONCURRENCY_NAME), SourceLoc()});
 }
 
+void CompilerInstance::verifyImplicitStringProcessingImport() {
+  if (Invocation.shouldImportSwiftStringProcessing() &&
+      !canImportSwiftStringProcessing()) {
+    Diagnostics.diagnose(SourceLoc(),
+                         diag::warn_implicit_string_processing_import_failed);
+  }
+}
+
+bool CompilerInstance::canImportSwiftStringProcessing() const {
+  return getASTContext().canImportModule(
+      {getASTContext().getIdentifier(SWIFT_STRING_PROCESSING_NAME),
+       SourceLoc()});
+}
+
 ImplicitImportInfo CompilerInstance::getImplicitImportInfo() const {
   auto &frontendOpts = Invocation.getFrontendOptions();
 
@@ -873,6 +891,19 @@ ImplicitImportInfo CompilerInstance::getImplicitImportInfo() const {
     }
   }
 
+  if (Invocation.shouldImportSwiftStringProcessing()) {
+    switch (imports.StdlibKind) {
+    case ImplicitStdlibKind::Builtin:
+    case ImplicitStdlibKind::None:
+      break;
+
+    case ImplicitStdlibKind::Stdlib:
+      if (canImportSwiftStringProcessing())
+        pushImport(SWIFT_STRING_PROCESSING_NAME);
+      break;
+    }
+  }
+
   imports.ShouldImportUnderlyingModule = frontendOpts.ImportUnderlyingModule;
   imports.BridgingHeaderPath = frontendOpts.ImplicitObjCHeaderPath;
   return imports;
@@ -1084,6 +1115,7 @@ bool CompilerInstance::loadStdlibIfNeeded() {
   }
 
   verifyImplicitConcurrencyImport();
+  verifyImplicitStringProcessingImport();
 
   // If we failed to load, we should have already diagnosed.
   if (M->failedToLoad()) {

lib/Parse/ParseRegex.cpp

@@ -54,30 +54,6 @@ ParserResult<Expr> Parser::parseExprRegexLiteral() {
     diagnose(Tok, diag::regex_literal_parsing_error, errorStr);
 
   auto loc = consumeToken();
-
-  // Create an implicit .init(_regexString: "<regex text>") call to serve as the
-  // semantic expression for the regex. The type-checker will provide it with
-  // the correct contextual type. We delay the contextual type for a couple of
-  // reasons:
-  // 1. We need to delay type lookup until after parsing.
-  // 2. Even if the AST synthesis were done lazily in e.g a request, we don't
-  //    currently have great support for implicit TypeExprs for unopened generic
-  //    types, as we want to phase out the use of UnboundGenericType. The Regex
-  //    type isn't currently generic, but might be in the future.
-  auto *regexStringExpr =
-      new (Context) StringLiteralExpr(Context.AllocateCopy(regexText), loc);
-  regexStringExpr->setImplicit();
-
-  DeclName initName(Context, DeclBaseName::createConstructor(),
-                    {Context.Id__regexString});
-  DeclNameRef initNameRef(initName);
-  auto *dotInit = new (Context) UnresolvedMemberExpr(
-      /*dotLoc*/ loc, DeclNameLoc(loc), initNameRef, /*implicit*/ true);
-
-  auto *args =
-      ArgumentList::forImplicitCallTo(initNameRef, {regexStringExpr}, Context);
-  auto *call = CallExpr::createImplicit(Context, dotInit, args);
-
   return makeParserResult(
-      RegexLiteralExpr::createParsed(Context, loc, regexText, call));
+      RegexLiteralExpr::createParsed(Context, loc, regexText));
 }

lib/SILGen/SILGenApply.cpp

@@ -1896,6 +1896,25 @@ buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
   return builtinLiteralArgs;
 }
 
+static inline PreparedArguments
+buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
+                        RegexLiteralExpr *expr) {
+  auto &ctx = SGF.getASTContext();
+  // %0 = string_literal <regex text>
+  auto strLiteralArgs = emitStringLiteral(SGF, expr, expr->getRegexText(), C,
+                                          StringLiteralExpr::Encoding::UTF8);
+  // %1 = function_ref String.init(
+  //   _builtinStringLiteral:utf8CodeUnitCount:isASCII:)
+  // %2 = apply %1(%0, ..., ...) -> $String
+  auto strInitDecl = ctx.getStringBuiltinInitDecl(ctx.getStringDecl());
+  RValue string = SGF.emitApplyAllocatingInitializer(
+      expr, strInitDecl, std::move(strLiteralArgs),
+      /*overriddenSelfType*/ Type(), SGFContext());
+  PreparedArguments args({AnyFunctionType::Param(ctx.getStringType())});
+  args.add(expr, std::move(string));
+  return args;
+}
+
 static inline PreparedArguments
 buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
                         MagicIdentifierLiteralExpr *magicLiteral) {
@@ -1961,6 +1980,8 @@ static inline PreparedArguments buildBuiltinLiteralArgs(SILGenFunction &SGF,
     return buildBuiltinLiteralArgs(SGF, C, integerLiteral);
   } else if (auto floatLiteral = dyn_cast<FloatLiteralExpr>(literal)) {
     return buildBuiltinLiteralArgs(SGF, C, floatLiteral);
+  } else if (auto regexLiteral = dyn_cast<RegexLiteralExpr>(literal)) {
+    return buildBuiltinLiteralArgs(SGF, C, regexLiteral);
   } else {
     return buildBuiltinLiteralArgs(
         SGF, C, cast<MagicIdentifierLiteralExpr>(literal));

lib/SILGen/SILGenExpr.cpp

@@ -2528,7 +2528,7 @@ visitInterpolatedStringLiteralExpr(InterpolatedStringLiteralExpr *E,
 }
 
 RValue RValueEmitter::visitRegexLiteralExpr(RegexLiteralExpr *E, SGFContext C) {
-  return SGF.emitRValue(E->getSemanticExpr());
+  return SGF.emitLiteral(E, C);
 }
 
 RValue RValueEmitter::

lib/Sema/CSApply.cpp

@@ -2792,7 +2792,10 @@ namespace {
     }
 
     Expr *visitRegexLiteralExpr(RegexLiteralExpr *expr) {
-      return simplifyExprType(expr);
+      simplifyExprType(expr);
+      expr->setInitializer(
+          cs.getASTContext().getRegexInitDecl(cs.getType(expr)));
+      return expr;
     }
 
     Expr *visitMagicIdentifierLiteralExpr(MagicIdentifierLiteralExpr *expr) {