Update regex literal lexing and emission

Update the lexing implementation to defer to the
regex library, which will pass back the pointer
from to resume lexing, and update the emission to
call the new `Regex(_regexString:version:)`
overload, that will accept the regex string with
delimiters.

Because this uses the library's lexing
implementation, the delimiters are now `'/.../'`
and `'|...|'` instead of plain `'...'`.

Author: hamishknight

include/swift/AST/ASTContext.h

@@ -621,7 +621,7 @@ class ASTContext final {
                                      KnownProtocolKind builtinProtocol,
                 llvm::function_ref<DeclName (ASTContext &ctx)> initName) const;
 
-  /// Retrieve _StringProcessing.Regex.init(_regexString: String).
+  /// Retrieve _StringProcessing.Regex.init(_regexString: String, version: Int).
   ConcreteDeclRef getRegexInitDecl(Type regexType) const;
 
   /// Retrieve the declaration of Swift.<(Int, Int) -> Bool.

include/swift/AST/Expr.h

@@ -966,18 +966,23 @@ class InterpolatedStringLiteralExpr : public LiteralExpr {
 class RegexLiteralExpr : public LiteralExpr {
   SourceLoc Loc;
   StringRef RegexText;
+  unsigned Version;
 
-  RegexLiteralExpr(SourceLoc loc, StringRef regexText, bool isImplicit)
+  RegexLiteralExpr(SourceLoc loc, StringRef regexText, unsigned version,
+                   bool isImplicit)
       : LiteralExpr(ExprKind::RegexLiteral, isImplicit), Loc(loc),
-        RegexText(regexText) {}
+        RegexText(regexText), Version(version) {}
 
 public:
   static RegexLiteralExpr *createParsed(ASTContext &ctx, SourceLoc loc,
-                                        StringRef regexText);
+                                        StringRef regexText, unsigned version);
 
   /// Retrieve the raw regex text.
   StringRef getRegexText() const { return RegexText; }
 
+  /// Retrieve the version of the regex string.
+  unsigned getVersion() const { return Version; }
+
   SourceRange getSourceRange() const { return Loc; }
 
   static bool classof(const Expr *E) {

include/swift/AST/KnownIdentifiers.def

@@ -252,6 +252,7 @@ IDENTIFIER(zero)
 // String processing
 IDENTIFIER(Regex)
 IDENTIFIER_(regexString)
+IDENTIFIER(version)
 IDENTIFIER_(StringProcessing)
 
 // Distributed actors

include/swift/Parse/ExperimentalRegexBridging.h

@@ -5,17 +5,43 @@
 extern "C" {
 #endif
 
-typedef const char *(* ParseRegexStrawperson)(const char *);
-
-void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn);
-bool Parser_hasParseRegexStrawperson();
+/// Attempt to lex a regex literal string. Takes the following arguments:
+///
+/// - CurPtrPtr: A pointer to the current pointer of lexer, which should be the
+///              start of the literal. This will be advanced to the point at
+///              which the lexer should resume, or will remain the same if this
+///              is not a regex literal.
+/// - BufferEnd: A pointer to the end of the buffer, which should not be lexed
+///              past.
+/// - ErrorOut: If an error is encountered, this will be set to the error
+///             string.
+///
+/// Returns: A bool indicating whether lexing was completely erroneous, and
+///          cannot be recovered from, or false if there either was no error,
+///          or there was a recoverable error.
+typedef bool(* RegexLiteralLexingFn)(/*CurPtrPtr*/ const char **,
+                                     /*BufferEnd*/ const char *,
+                                     /*ErrorOut*/ const char **);
+void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn);
+
+/// Parse a regex literal string. Takes the following arguments:
+///
+/// - InputPtr: A null-terminated C string of the regex literal.
+/// - ErrorOut: A buffer accepting an error string upon error.
+/// - VersionOut: A buffer accepting a regex literal format version.
+/// - CaptureStructureOut: A buffer accepting a byte sequence representing the
+///                        capture structure of the literal.
+/// - CaptureStructureSize: The size of the capture structure buffer. Must be
+///                         greater than or equal to `strlen(InputPtr)`.
+typedef void(* RegexLiteralParsingFn)(/*InputPtr*/ const char *,
+                                      /*ErrorOut*/ const char **,
+                                      /*VersionOut*/ unsigned *,
+                                      /*CaptureStructureOut*/ char *,
+                                      /*CaptureStructureSize*/ unsigned);
+void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn);
 
 #ifdef __cplusplus
 } // extern "C"
 #endif
 
 #endif // EXPERIMENTAL_REGEX_BRIDGING
-
-
-//const char* experimental_regex_strawperson(const char *in);
-

include/swift/Parse/Lexer.h

@@ -595,7 +595,9 @@ class Lexer {
   void lexStringLiteral(unsigned CustomDelimiterLen = 0);
   void lexEscapedIdentifier();
 
-  void lexRegexLiteral(const char *TokStart);
+  /// Attempt to lex a regex literal, returning true if a regex literal was
+  /// lexed, false if this is not a regex literal.
+  bool tryLexRegexLiteral(const char *TokStart);
 
   void tryLexEditorPlaceholder();
   const char *findEndOfCurlyQuoteStringLiteral(const char *,
@@ -612,9 +614,6 @@ class Lexer {
 
   /// Emit diagnostics for single-quote string and suggest replacement
   /// with double-quoted equivalent.
-  ///
-  /// Or, if we're in strawperson mode, we will emit a custom
-  /// error message instead, determined by the Swift library.
   void diagnoseSingleQuoteStringLiteral(const char *TokStart,
                                         const char *TokEnd);
 

lib/AST/ASTContext.cpp

@@ -1221,7 +1221,7 @@ ConcreteDeclRef ASTContext::getRegexInitDecl(Type regexType) const {
   auto *spModule = getLoadedModule(Id_StringProcessing);
   DeclName name(*const_cast<ASTContext *>(this),
                 DeclBaseName::createConstructor(),
-                {Id_regexString});
+                {Id_regexString, Id_version});
   SmallVector<ValueDecl *, 1> results;
   spModule->lookupQualified(getRegexType(), DeclNameRef(name),
                             NL_IncludeUsableFromInline, results);

lib/AST/Expr.cpp

@@ -2246,8 +2246,9 @@ SourceLoc TapExpr::getEndLoc() const {
 
 RegexLiteralExpr *
 RegexLiteralExpr::createParsed(ASTContext &ctx, SourceLoc loc,
-                               StringRef regexText) {
-  return new (ctx) RegexLiteralExpr(loc, regexText, /*implicit*/ false);
+                               StringRef regexText, unsigned version) {
+  return new (ctx) RegexLiteralExpr(loc, regexText, version,
+                                    /*implicit*/ false);
 }
 
 void swift::simple_display(llvm::raw_ostream &out, const ClosureExpr *CE) {

lib/Parse/Lexer.cpp

@@ -33,6 +33,13 @@
 
 #include <limits>
 
+// Regex lexing delivered via libSwift.
+#include "swift/Parse/ExperimentalRegexBridging.h"
+static RegexLiteralLexingFn regexLiteralLexingFn = nullptr;
+void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn) {
+  regexLiteralLexingFn = fn;
+}
+
 using namespace swift;
 using namespace swift::syntax;
 
@@ -1951,36 +1958,46 @@ const char *Lexer::findEndOfCurlyQuoteStringLiteral(const char *Body,
   }
 }
 
-void Lexer::lexRegexLiteral(const char *TokStart) {
+bool Lexer::tryLexRegexLiteral(const char *TokStart) {
   assert(*TokStart == '\'');
 
-  bool HadError = false;
-  while (true) {
-    // Check if we reached the end of the literal without terminating.
-    if (CurPtr >= BufferEnd || *CurPtr == '\n' || *CurPtr == '\r') {
-      diagnose(TokStart, diag::lex_unterminated_regex);
-      return formToken(tok::unknown, TokStart);
-    }
+  // We need to have experimental string processing enabled, and have the
+  // parsing logic for regex literals available.
+  if (!LangOpts.EnableExperimentalStringProcessing || !regexLiteralLexingFn)
+    return false;
 
-    const auto *CharStart = CurPtr;
-    uint32_t CharValue = validateUTF8CharacterAndAdvance(CurPtr, BufferEnd);
-    if (CharValue == ~0U) {
-      diagnose(CharStart, diag::lex_invalid_utf8);
-      HadError = true;
-      continue;
-    }
-    if (CharValue == '\\' && (*CurPtr == '\'' || *CurPtr == '\\')) {
-      // Skip escaped delimiter or \.
-      CurPtr++;
-    } else if (CharValue == '\'') {
-      // End of literal, stop.
-      break;
-    }
+  // Ask libswift to try and lex a regex literal.
+  // - Ptr will not be advanced if this is not for a regex literal.
+  // - ErrStr will be set if there is any error to emit.
+  // - CompletelyErroneous will be set if there was an error that cannot be
+  //   recovered from.
+  auto *Ptr = TokStart;
+  const char *ErrStr = nullptr;
+  bool CompletelyErroneous = regexLiteralLexingFn(&Ptr, BufferEnd, &ErrStr);
+  if (ErrStr)
+    diagnose(TokStart, diag::regex_literal_parsing_error, ErrStr);
+
+  // If we didn't make any lexing progress, this isn't a regex literal and we
+  // should fallback to lexing as something else.
+  if (Ptr == TokStart)
+    return false;
+
+  // Update to point to where we ended regex lexing.
+  assert(Ptr > TokStart && Ptr <= BufferEnd);
+  CurPtr = Ptr;
+
+  // If the lexing was completely erroneous, form an unknown token.
+  if (CompletelyErroneous) {
+    assert(ErrStr);
+    formToken(tok::unknown, TokStart);
+    return true;
   }
-  if (HadError)
-    return formToken(tok::unknown, TokStart);
 
+  // Otherwise, we either had a successful lex, or something that was
+  // recoverable.
+  assert(ErrStr || CurPtr[-1] == '\'');
   formToken(tok::regex_literal, TokStart);
+  return true;
 }
 
 /// lexEscapedIdentifier:
@@ -2528,11 +2545,11 @@ void Lexer::lexImpl() {
 
   case '\'':
     // If we have experimental string processing enabled, and have the parsing
-    // logic for regex literals, lex a single quoted string as a regex literal.
-    if (LangOpts.EnableExperimentalStringProcessing &&
-        Parser_hasParseRegexStrawperson()) {
-      return lexRegexLiteral(TokStart);
-    }
+    // logic for regex literals, try to lex a single quoted string as a regex
+    // literal.
+    if (tryLexRegexLiteral(TokStart))
+      return;
+
     // Otherwise lex as a string literal and emit a diagnostic.
     LLVM_FALLTHROUGH;
   case '"':

lib/Parse/ParseRegex.cpp

@@ -22,38 +22,36 @@
 
 // Regex parser delivered via libSwift
 #include "swift/Parse/ExperimentalRegexBridging.h"
-static ParseRegexStrawperson parseRegexStrawperson = nullptr;
-void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn) {
-  parseRegexStrawperson = fn;
-}
-// Exposes the presence of the regex parsing function to the lexer.
-bool Parser_hasParseRegexStrawperson() {
-  return parseRegexStrawperson != nullptr;
+static RegexLiteralParsingFn regexLiteralParsingFn = nullptr;
+void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn) {
+  regexLiteralParsingFn = fn;
 }
 
 using namespace swift;
 using namespace swift::syntax;
 
 ParserResult<Expr> Parser::parseExprRegexLiteral() {
   assert(Tok.is(tok::regex_literal));
-  assert(parseRegexStrawperson);
+  assert(regexLiteralParsingFn);
 
   SyntaxParsingContext LocalContext(SyntaxContext,
                                     SyntaxKind::RegexLiteralExpr);
-  // Strip off delimiters.
-  auto rawText = Tok.getText();
-  assert(rawText.front() == '\'' && rawText.back() == '\'');
-  auto regexText = rawText.slice(1, rawText.size() - 1);
+
+  auto regexText = Tok.getText();
 
   // Let the Swift library parse the contents, returning an error, or null if
   // successful.
   // TODO: We need to be able to pass back a source location to emit the error
   // at.
-  auto *errorStr = parseRegexStrawperson(regexText.str().c_str());
+  const char *errorStr = nullptr;
+  unsigned version;
+  regexLiteralParsingFn(regexText.str().c_str(), &errorStr, &version,
+                        /*captureStructureOut*/ nullptr,
+                        /*captureStructureSize*/ 0);
   if (errorStr)
     diagnose(Tok, diag::regex_literal_parsing_error, errorStr);
 
   auto loc = consumeToken();
   return makeParserResult(
-      RegexLiteralExpr::createParsed(Context, loc, regexText));
+      RegexLiteralExpr::createParsed(Context, loc, regexText, version));
 }

lib/SILGen/SILGenApply.cpp

@@ -1910,10 +1910,31 @@ buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
   RValue string = SGF.emitApplyAllocatingInitializer(
       expr, strInitDecl, std::move(strLiteralArgs),
       /*overriddenSelfType*/ Type(), SGFContext());
-  PreparedArguments args(
-      ArrayRef<AnyFunctionType::Param>({
-          AnyFunctionType::Param(ctx.getStringType())}));
+
+  // The version of the regex string.
+  // %3 = integer_literal $Builtin.IntLiteral <version>
+  auto versionIntLiteral =
+      ManagedValue::forUnmanaged(SGF.B.createIntegerLiteral(
+          expr, SILType::getBuiltinIntegerLiteralType(SGF.getASTContext()),
+          expr->getVersion()));
+
+  using Param = AnyFunctionType::Param;
+  auto builtinIntTy = versionIntLiteral.getType().getASTType();
+  PreparedArguments versionIntBuiltinArgs(ArrayRef<Param>{Param(builtinIntTy)});
+  versionIntBuiltinArgs.add(
+      expr, RValue(SGF, {versionIntLiteral}, builtinIntTy));
+
+  // %4 = function_ref Int.init(_builtinIntegerLiteral: Builtin.IntLiteral)
+  // %5 = apply %5(%3, ...) -> $Int
+  auto intLiteralInit = ctx.getIntBuiltinInitDecl(ctx.getIntDecl());
+  RValue versionInt = SGF.emitApplyAllocatingInitializer(
+      expr, intLiteralInit, std::move(versionIntBuiltinArgs),
+      /*overriddenSelfType*/ Type(), SGFContext());
+
+  PreparedArguments args(ArrayRef<Param>{Param(ctx.getStringType()),
+                                         Param(ctx.getIntType())});
   args.add(expr, std::move(string));
+  args.add(expr, std::move(versionInt));
   return args;
 }
 

libswift/Sources/ExperimentalRegex/Regex.swift

@@ -1,30 +1,6 @@
 import ExperimentalRegexBridging
 
-public func experimental_regex_strawperson(
-  _ ptr: UnsafePointer<CChar>?
-) -> UnsafePointer<CChar>? {
-  guard let s = ptr else { return nil }
-
-  func makeError(_ str: String) -> UnsafePointer<CChar>? {
-    let count = str.utf8.count + 1
-    return str.withCString {
-      assert($0[count-1] == 0)
-      let ptr = UnsafeMutablePointer<CChar>.allocate(capacity: count)
-      ptr.initialize(from: $0, count: count)
-      return UnsafePointer(ptr)
-    }
-  }
-
-  let str = String(cString: s)
-  do {
-    let _ = try parse(str, .traditional)
-    return nil
-  } catch {
-    return makeError(
-      "cannot parse regular expression: \(String(describing: error))")
-  }
-}
-
 public func registerRegexParser() {
-  Parser_registerParseRegexStrawperson({ experimental_regex_strawperson($0) })
+  Parser_registerRegexLiteralParsingFn(libswiftParseRegexLiteral)
+  Parser_registerRegexLiteralLexingFn(libswiftLexRegexLiteral)
 }

test/StringProcessing/Parse/regex.swift

@@ -1,10 +1,10 @@
 // RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
 // REQUIRES: libswift
 
-_ = 'abc'
+_ = '/abc/'
 
-_ = ('[*', '+]', '.]')
+_ = ('/[*/', '/+]/', '/.]/')
 // expected-error@-1 {{cannot parse regular expression}}
 
-_ = '\w+'
-_ = '\'\\'
+_ = '/\w+/'
+_ = '/\'\\/'

test/StringProcessing/Parse/regex_parse_end_of_buffer.swift

@@ -0,0 +1,6 @@
+// RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
+// REQUIRES: libswift
+
+// Note there is purposefully no trailing newline here.
+// expected-error@+1 {{unterminated regex literal}}
+var unterminated = '/xy

test/StringProcessing/Parse/regex_parse_error.swift

@@ -1,7 +1,7 @@
 // RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
 // REQUIRES: libswift
 
-let s = '\\'' // expected-error {{unterminated regex literal}}
+let s = '/\\/''/ // expected-error {{unterminated regex literal}}
 
 // expected-error@+1 {{unterminated regex literal}}
-var unterminated = 'xy
+var unterminated = '/xy