Revert "Merge pull request #40595 from hamishknight/straw-bales"

This reverts commit a67a043, reversing
changes made to 9965df7.

This commit or the earlier commit this commit is based on (#40531) broke the
incremental bot.

Author: aschwaighofer

cmake/modules/AddSwift.cmake

@@ -634,13 +634,8 @@ function(add_libswift_module module)
                         ""
                         "DEPENDS"
                         ${ARGN})
-  set(raw_sources ${ALSM_UNPARSED_ARGUMENTS})
-  set(sources)
-  foreach(raw_source ${raw_sources})
-    get_filename_component(
-      raw_source "${raw_source}" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
-    list(APPEND sources "${raw_source}")
-  endforeach()
+  set(sources ${ALSM_UNPARSED_ARGUMENTS})
+  list(TRANSFORM sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
   set(target_name "LibSwift${module}")
 

include/swift/AST/ASTContext.h

@@ -621,7 +621,7 @@ class ASTContext final {
                                      KnownProtocolKind builtinProtocol,
                 llvm::function_ref<DeclName (ASTContext &ctx)> initName) const;
 
-  /// Retrieve _StringProcessing.Regex.init(_regexString: String, version: Int).
+  /// Retrieve _StringProcessing.Regex.init(_regexString: String).
   ConcreteDeclRef getRegexInitDecl(Type regexType) const;
 
   /// Retrieve the declaration of Swift.<(Int, Int) -> Bool.

include/swift/AST/Expr.h

@@ -966,23 +966,18 @@ class InterpolatedStringLiteralExpr : public LiteralExpr {
 class RegexLiteralExpr : public LiteralExpr {
   SourceLoc Loc;
   StringRef RegexText;
-  unsigned Version;
 
-  RegexLiteralExpr(SourceLoc loc, StringRef regexText, unsigned version,
-                   bool isImplicit)
+  RegexLiteralExpr(SourceLoc loc, StringRef regexText, bool isImplicit)
       : LiteralExpr(ExprKind::RegexLiteral, isImplicit), Loc(loc),
-        RegexText(regexText), Version(version) {}
+        RegexText(regexText) {}
 
 public:
   static RegexLiteralExpr *createParsed(ASTContext &ctx, SourceLoc loc,
-                                        StringRef regexText, unsigned version);
+                                        StringRef regexText);
 
   /// Retrieve the raw regex text.
   StringRef getRegexText() const { return RegexText; }
 
-  /// Retrieve the version of the regex string.
-  unsigned getVersion() const { return Version; }
-
   SourceRange getSourceRange() const { return Loc; }
 
   static bool classof(const Expr *E) {

include/swift/AST/KnownIdentifiers.def

@@ -252,7 +252,6 @@ IDENTIFIER(zero)
 // String processing
 IDENTIFIER(Regex)
 IDENTIFIER_(regexString)
-IDENTIFIER(version)
 IDENTIFIER_(StringProcessing)
 
 // Distributed actors

include/swift/Parse/ExperimentalRegexBridging.h

@@ -5,43 +5,17 @@
 extern "C" {
 #endif
 
-/// Attempt to lex a regex literal string. Takes the following arguments:
-///
-/// - CurPtrPtr: A pointer to the current pointer of lexer, which should be the
-///              start of the literal. This will be advanced to the point at
-///              which the lexer should resume, or will remain the same if this
-///              is not a regex literal.
-/// - BufferEnd: A pointer to the end of the buffer, which should not be lexed
-///              past.
-/// - ErrorOut: If an error is encountered, this will be set to the error
-///             string.
-///
-/// Returns: A bool indicating whether lexing was completely erroneous, and
-///          cannot be recovered from, or false if there either was no error,
-///          or there was a recoverable error.
-typedef bool(* RegexLiteralLexingFn)(/*CurPtrPtr*/ const char **,
-                                     /*BufferEnd*/ const char *,
-                                     /*ErrorOut*/ const char **);
-void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn);
-
-/// Parse a regex literal string. Takes the following arguments:
-///
-/// - InputPtr: A null-terminated C string of the regex literal.
-/// - ErrorOut: A buffer accepting an error string upon error.
-/// - VersionOut: A buffer accepting a regex literal format version.
-/// - CaptureStructureOut: A buffer accepting a byte sequence representing the
-///                        capture structure of the literal.
-/// - CaptureStructureSize: The size of the capture structure buffer. Must be
-///                         greater than or equal to `strlen(InputPtr)`.
-typedef void(* RegexLiteralParsingFn)(/*InputPtr*/ const char *,
-                                      /*ErrorOut*/ const char **,
-                                      /*VersionOut*/ unsigned *,
-                                      /*CaptureStructureOut*/ char *,
-                                      /*CaptureStructureSize*/ unsigned);
-void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn);
+typedef const char *(* ParseRegexStrawperson)(const char *);
+
+void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn);
+bool Parser_hasParseRegexStrawperson();
 
 #ifdef __cplusplus
 } // extern "C"
 #endif
 
 #endif // EXPERIMENTAL_REGEX_BRIDGING
+
+
+//const char* experimental_regex_strawperson(const char *in);
+

include/swift/Parse/Lexer.h

@@ -595,9 +595,7 @@ class Lexer {
   void lexStringLiteral(unsigned CustomDelimiterLen = 0);
   void lexEscapedIdentifier();
 
-  /// Attempt to lex a regex literal, returning true if a regex literal was
-  /// lexed, false if this is not a regex literal.
-  bool tryLexRegexLiteral(const char *TokStart);
+  void lexRegexLiteral(const char *TokStart);
 
   void tryLexEditorPlaceholder();
   const char *findEndOfCurlyQuoteStringLiteral(const char *,
@@ -614,6 +612,9 @@ class Lexer {
 
   /// Emit diagnostics for single-quote string and suggest replacement
   /// with double-quoted equivalent.
+  ///
+  /// Or, if we're in strawperson mode, we will emit a custom
+  /// error message instead, determined by the Swift library.
   void diagnoseSingleQuoteStringLiteral(const char *TokStart,
                                         const char *TokEnd);
 

lib/AST/ASTContext.cpp

@@ -1221,7 +1221,7 @@ ConcreteDeclRef ASTContext::getRegexInitDecl(Type regexType) const {
   auto *spModule = getLoadedModule(Id_StringProcessing);
   DeclName name(*const_cast<ASTContext *>(this),
                 DeclBaseName::createConstructor(),
-                {Id_regexString, Id_version});
+                {Id_regexString});
   SmallVector<ValueDecl *, 1> results;
   spModule->lookupQualified(getRegexType(), DeclNameRef(name),
                             NL_IncludeUsableFromInline, results);

lib/AST/Expr.cpp

@@ -2246,9 +2246,8 @@ SourceLoc TapExpr::getEndLoc() const {
 
 RegexLiteralExpr *
 RegexLiteralExpr::createParsed(ASTContext &ctx, SourceLoc loc,
-                               StringRef regexText, unsigned version) {
-  return new (ctx) RegexLiteralExpr(loc, regexText, version,
-                                    /*implicit*/ false);
+                               StringRef regexText) {
+  return new (ctx) RegexLiteralExpr(loc, regexText, /*implicit*/ false);
 }
 
 void swift::simple_display(llvm::raw_ostream &out, const ClosureExpr *CE) {

lib/Parse/Lexer.cpp

@@ -33,13 +33,6 @@
 
 #include <limits>
 
-// Regex lexing delivered via libSwift.
-#include "swift/Parse/ExperimentalRegexBridging.h"
-static RegexLiteralLexingFn regexLiteralLexingFn = nullptr;
-void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn) {
-  regexLiteralLexingFn = fn;
-}
-
 using namespace swift;
 using namespace swift::syntax;
 
@@ -1958,46 +1951,36 @@ const char *Lexer::findEndOfCurlyQuoteStringLiteral(const char *Body,
   }
 }
 
-bool Lexer::tryLexRegexLiteral(const char *TokStart) {
+void Lexer::lexRegexLiteral(const char *TokStart) {
   assert(*TokStart == '\'');
 
-  // We need to have experimental string processing enabled, and have the
-  // parsing logic for regex literals available.
-  if (!LangOpts.EnableExperimentalStringProcessing || !regexLiteralLexingFn)
-    return false;
-
-  // Ask libswift to try and lex a regex literal.
-  // - Ptr will not be advanced if this is not for a regex literal.
-  // - ErrStr will be set if there is any error to emit.
-  // - CompletelyErroneous will be set if there was an error that cannot be
-  //   recovered from.
-  auto *Ptr = TokStart;
-  const char *ErrStr = nullptr;
-  bool CompletelyErroneous = regexLiteralLexingFn(&Ptr, BufferEnd, &ErrStr);
-  if (ErrStr)
-    diagnose(TokStart, diag::regex_literal_parsing_error, ErrStr);
-
-  // If we didn't make any lexing progress, this isn't a regex literal and we
-  // should fallback to lexing as something else.
-  if (Ptr == TokStart)
-    return false;
-
-  // Update to point to where we ended regex lexing.
-  assert(Ptr > TokStart && Ptr <= BufferEnd);
-  CurPtr = Ptr;
+  bool HadError = false;
+  while (true) {
+    // Check if we reached the end of the literal without terminating.
+    if (CurPtr >= BufferEnd || *CurPtr == '\n' || *CurPtr == '\r') {
+      diagnose(TokStart, diag::lex_unterminated_regex);
+      return formToken(tok::unknown, TokStart);
+    }
 
-  // If the lexing was completely erroneous, form an unknown token.
-  if (CompletelyErroneous) {
-    assert(ErrStr);
-    formToken(tok::unknown, TokStart);
-    return true;
+    const auto *CharStart = CurPtr;
+    uint32_t CharValue = validateUTF8CharacterAndAdvance(CurPtr, BufferEnd);
+    if (CharValue == ~0U) {
+      diagnose(CharStart, diag::lex_invalid_utf8);
+      HadError = true;
+      continue;
+    }
+    if (CharValue == '\\' && (*CurPtr == '\'' || *CurPtr == '\\')) {
+      // Skip escaped delimiter or \.
+      CurPtr++;
+    } else if (CharValue == '\'') {
+      // End of literal, stop.
+      break;
+    }
   }
+  if (HadError)
+    return formToken(tok::unknown, TokStart);
 
-  // Otherwise, we either had a successful lex, or something that was
-  // recoverable.
-  assert(ErrStr || CurPtr[-1] == '\'');
   formToken(tok::regex_literal, TokStart);
-  return true;
 }
 
 /// lexEscapedIdentifier:
@@ -2545,11 +2528,11 @@ void Lexer::lexImpl() {
 
   case '\'':
     // If we have experimental string processing enabled, and have the parsing
-    // logic for regex literals, try to lex a single quoted string as a regex
-    // literal.
-    if (tryLexRegexLiteral(TokStart))
-      return;
-
+    // logic for regex literals, lex a single quoted string as a regex literal.
+    if (LangOpts.EnableExperimentalStringProcessing &&
+        Parser_hasParseRegexStrawperson()) {
+      return lexRegexLiteral(TokStart);
+    }
     // Otherwise lex as a string literal and emit a diagnostic.
     LLVM_FALLTHROUGH;
   case '"':

lib/Parse/ParseRegex.cpp

@@ -22,36 +22,38 @@
 
 // Regex parser delivered via libSwift
 #include "swift/Parse/ExperimentalRegexBridging.h"
-static RegexLiteralParsingFn regexLiteralParsingFn = nullptr;
-void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn) {
-  regexLiteralParsingFn = fn;
+static ParseRegexStrawperson parseRegexStrawperson = nullptr;
+void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn) {
+  parseRegexStrawperson = fn;
+}
+// Exposes the presence of the regex parsing function to the lexer.
+bool Parser_hasParseRegexStrawperson() {
+  return parseRegexStrawperson != nullptr;
 }
 
 using namespace swift;
 using namespace swift::syntax;
 
 ParserResult<Expr> Parser::parseExprRegexLiteral() {
   assert(Tok.is(tok::regex_literal));
-  assert(regexLiteralParsingFn);
+  assert(parseRegexStrawperson);
 
   SyntaxParsingContext LocalContext(SyntaxContext,
                                     SyntaxKind::RegexLiteralExpr);
-
-  auto regexText = Tok.getText();
+  // Strip off delimiters.
+  auto rawText = Tok.getText();
+  assert(rawText.front() == '\'' && rawText.back() == '\'');
+  auto regexText = rawText.slice(1, rawText.size() - 1);
 
   // Let the Swift library parse the contents, returning an error, or null if
   // successful.
   // TODO: We need to be able to pass back a source location to emit the error
   // at.
-  const char *errorStr = nullptr;
-  unsigned version;
-  regexLiteralParsingFn(regexText.str().c_str(), &errorStr, &version,
-                        /*captureStructureOut*/ nullptr,
-                        /*captureStructureSize*/ 0);
+  auto *errorStr = parseRegexStrawperson(regexText.str().c_str());
   if (errorStr)
     diagnose(Tok, diag::regex_literal_parsing_error, errorStr);
 
   auto loc = consumeToken();
   return makeParserResult(
-      RegexLiteralExpr::createParsed(Context, loc, regexText, version));
+      RegexLiteralExpr::createParsed(Context, loc, regexText));
 }

lib/SILGen/SILGenApply.cpp

@@ -1910,31 +1910,10 @@ buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
   RValue string = SGF.emitApplyAllocatingInitializer(
       expr, strInitDecl, std::move(strLiteralArgs),
       /*overriddenSelfType*/ Type(), SGFContext());
-
-  // The version of the regex string.
-  // %3 = integer_literal $Builtin.IntLiteral <version>
-  auto versionIntLiteral =
-      ManagedValue::forUnmanaged(SGF.B.createIntegerLiteral(
-          expr, SILType::getBuiltinIntegerLiteralType(SGF.getASTContext()),
-          expr->getVersion()));
-
-  using Param = AnyFunctionType::Param;
-  auto builtinIntTy = versionIntLiteral.getType().getASTType();
-  PreparedArguments versionIntBuiltinArgs(ArrayRef<Param>{Param(builtinIntTy)});
-  versionIntBuiltinArgs.add(
-      expr, RValue(SGF, {versionIntLiteral}, builtinIntTy));
-
-  // %4 = function_ref Int.init(_builtinIntegerLiteral: Builtin.IntLiteral)
-  // %5 = apply %5(%3, ...) -> $Int
-  auto intLiteralInit = ctx.getIntBuiltinInitDecl(ctx.getIntDecl());
-  RValue versionInt = SGF.emitApplyAllocatingInitializer(
-      expr, intLiteralInit, std::move(versionIntBuiltinArgs),
-      /*overriddenSelfType*/ Type(), SGFContext());
-
-  PreparedArguments args(ArrayRef<Param>{Param(ctx.getStringType()),
-                                         Param(ctx.getIntType())});
+  PreparedArguments args(
+      ArrayRef<AnyFunctionType::Param>({
+          AnyFunctionType::Param(ctx.getStringType())}));
   args.add(expr, std::move(string));
-  args.add(expr, std::move(versionInt));
   return args;
 }
 

lib/Sema/CSGen.cpp

@@ -1266,15 +1266,18 @@ namespace {
                            ctx.Id_Regex.str());
         return Type();
       }
+      auto substringType = ctx.getSubstringType();
       auto dynCapturesType = ctx.getDynamicCapturesType();
       if (!dynCapturesType) {
         ctx.Diags.diagnose(E->getLoc(),
                            diag::string_processing_lib_missing,
                            "DynamicCaptures");
         return Type();
       }
-      // TODO: Replace `DynamicCaptures` with type inferred from the regex.
-      return BoundGenericStructType::get(regexDecl, Type(), {dynCapturesType});
+      // TODO: Replace `(Substring, DynamicCaptures)` with type inferred from
+      // the regex.
+      auto matchType = TupleType::get({substringType, dynCapturesType}, ctx);
+      return BoundGenericStructType::get(regexDecl, Type(), {matchType});
     }
 
     Type visitDeclRefExpr(DeclRefExpr *E) {

libswift/Sources/CMakeLists.txt

@@ -6,8 +6,6 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
-  add_subdirectory(ExperimentalRegex)
-endif()
+add_subdirectory(ExperimentalRegex)
 add_subdirectory(SIL)
 add_subdirectory(Optimizer)

libswift/Sources/ExperimentalRegex/CMakeLists.txt

@@ -6,16 +6,7 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-file(GLOB_RECURSE _LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES
-  "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
-set(LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES)
-foreach(source ${_LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES})
-  file(TO_CMAKE_PATH "${source}" source)
-  list(APPEND LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES ${source})
-endforeach()
-message(STATUS "Using Experimental String Processing library for libswift ExperimentalRegex (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
-
 add_libswift_module(ExperimentalRegex
-  "${LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES}"
-  Regex.swift)
+  Regex.swift
+)