[Parse] Introduce /.../ regex literals

include/swift/AST/DiagnosticEngine.h

@@ -848,6 +848,7 @@ namespace swift {
     friend class DiagnosticTransaction;
     friend class CompoundDiagnosticTransaction;
     friend class DiagnosticStateRAII;
+    friend class DiagnosticQueue;
 
   public:
     explicit DiagnosticEngine(SourceManager &SourceMgr)
@@ -1137,10 +1138,20 @@ namespace swift {
     /// Send \c diag to all diagnostic consumers.
     void emitDiagnostic(const Diagnostic &diag);
 
+    /// Handle a new diagnostic, which will either be emitted, or added to an
+    /// active transaction.
+    void handleDiagnostic(Diagnostic &&diag);
+
+    /// Clear any tentative diagnostics.
+    void clearTentativeDiagnostics();
+
     /// Send all tentative diagnostics to all diagnostic consumers and
     /// delete them.
     void emitTentativeDiagnostics();
 
+    /// Forward all tentative diagnostics to a different diagnostic engine.
+    void forwardTentativeDiagnosticsTo(DiagnosticEngine &targetEngine);
+
   public:
     DiagnosticKind declaredDiagnosticKindFor(const DiagID id);
 
@@ -1333,6 +1344,78 @@ namespace swift {
     }
   };
 
+  /// Represents a queue of diagnostics that have their emission delayed until
+  /// the queue is destroyed. This is similar to DiagnosticTransaction, but
+  /// with a few key differences:
+  /// 
+  /// - The queue maintains its own diagnostic engine (which may be accessed
+  ///   through `getDiags()`), and diagnostics must be specifically emitted
+  ///   using that engine to be enqueued.
+  /// - It allows for non-LIFO transactions, as each queue operates
+  ///   independently.
+  /// - A queue can be drained multiple times without having to be recreated
+  ///   (unlike DiagnosticTransaction, it has no concept of "closing").
+  ///
+  /// Note you may add DiagnosticTransactions to the queue's diagnostic engine,
+  /// but they must be closed before attempting to clear or emit the diagnostics
+  /// in the queue.
+  ///
+  class DiagnosticQueue final {
+    /// The underlying diagnostic engine that the diagnostics will be emitted
+    /// by.
+    DiagnosticEngine &UnderlyingEngine;
+
+    /// A temporary engine used to queue diagnostics.
+    DiagnosticEngine QueueEngine;
+
+    /// Whether the queued diagnostics should be emitted on the destruction of
+    /// the queue, or whether they should be cleared.
+    bool EmitOnDestruction;
+
+  public:
+    DiagnosticQueue(const DiagnosticQueue &) = delete;
+    DiagnosticQueue &operator=(const DiagnosticQueue &) = delete;
+
+    /// Create a new diagnostic queue with a given engine to forward the
+    /// diagnostics to.
+    explicit DiagnosticQueue(DiagnosticEngine &engine, bool emitOnDestruction)
+        : UnderlyingEngine(engine), QueueEngine(engine.SourceMgr),
+          EmitOnDestruction(emitOnDestruction) {
+      // Open a transaction to avoid emitting any diagnostics for the temporary
+      // engine.
+      QueueEngine.TransactionCount++;
+    }
+
+    /// Retrieve the engine which may be used to enqueue diagnostics.
+    DiagnosticEngine &getDiags() { return QueueEngine; }
+
+    /// Retrieve the underlying engine which will receive the diagnostics.
+    DiagnosticEngine &getUnderlyingDiags() { return UnderlyingEngine; }
+
+    /// Clear this queue and erase all diagnostics recorded.
+    void clear() {
+      assert(QueueEngine.TransactionCount == 1 &&
+             "Must close outstanding DiagnosticTransactions before draining");
+      QueueEngine.clearTentativeDiagnostics();
+    }
+
+    /// Emit all the diagnostics recorded by this queue.
+    void emit() {
+      assert(QueueEngine.TransactionCount == 1 &&
+             "Must close outstanding DiagnosticTransactions before draining");
+      QueueEngine.forwardTentativeDiagnosticsTo(UnderlyingEngine);
+    }
+
+    ~DiagnosticQueue() {
+      if (EmitOnDestruction) {
+        emit();
+      } else {
+        clear();
+      }
+      QueueEngine.TransactionCount--;
+    }
+  };
+
   inline void
   DiagnosticEngine::diagnoseWithNotes(InFlightDiagnostic parentDiag,
                                       llvm::function_ref<void(void)> builder) {

include/swift/AST/DiagnosticsParse.def

@@ -94,6 +94,9 @@ ERROR(forbidden_extended_escaping_string,none,
 ERROR(regex_literal_parsing_error,none,
       "%0", (StringRef))
 
+ERROR(prefix_slash_not_allowed,none,
+      "prefix operator may not contain '/'", ())
+
 //------------------------------------------------------------------------------
 // MARK: Lexer diagnostics
 //------------------------------------------------------------------------------
@@ -140,7 +143,10 @@ ERROR(lex_invalid_escape_delimiter,none,
 ERROR(lex_invalid_closing_delimiter,none,
       "too many '#' characters in closing delimiter", ())
 
-ERROR(lex_unterminated_regex,none,
+ERROR(lex_regex_literal_invalid_starting_char,none,
+      "regex literal may not start with %0; add backslash to escape",
+      (StringRef))
+ERROR(lex_regex_literal_unterminated,none,
       "unterminated regex literal", ())
 
 ERROR(lex_invalid_unicode_scalar,none,

include/swift/Basic/LangOptions.h

@@ -564,8 +564,9 @@ namespace swift {
     /// Enables dumping type witness systems from associated type inference.
     bool DumpTypeWitnessSystems = false;
 
-    /// Enables `/.../` syntax regular-expression literals
-    bool EnableForwardSlashRegexLiterals = false;
+    /// Enables `/.../` syntax regular-expression literals. This requires
+    /// experimental string processing. Note this does not affect `#/.../#`.
+    bool EnableBareSlashRegexLiterals = false;
 
     /// Sets the target we are building for and updates platform conditions
     /// to match.

include/swift/Parse/Lexer.h

@@ -61,6 +61,16 @@ enum class LexerMode {
   SIL
 };
 
+/// Whether or not the lexer should attempt to lex a `/.../` regex literal.
+enum class LexerForwardSlashRegexMode {
+  /// No `/.../` regex literals will be lexed.
+  None,
+  /// A `/.../` regex literal will be lexed, but only if successful.
+  Tentative,
+  /// A `/.../` regex literal will always be lexed for a '/' character.
+  Always
+};
+
 /// Kinds of conflict marker which the lexer might encounter.
 enum class ConflictMarkerKind {
   /// A normal or diff3 conflict marker, initiated by at least 7 "<"s,
@@ -75,7 +85,10 @@ class Lexer {
   const LangOptions &LangOpts;
   const SourceManager &SourceMgr;
   const unsigned BufferID;
-  DiagnosticEngine *Diags;
+
+  /// A queue of diagnostics to emit when a token is consumed. We want to queue
+  /// them, as the parser may backtrack and re-lex a token.
+  Optional<DiagnosticQueue> DiagQueue;
 
   using State = LexerState;
 
@@ -109,6 +122,10 @@ class Lexer {
   /// a .sil file.
   const LexerMode LexMode;
 
+  /// Whether or not a `/.../` literal will be lexed.
+  LexerForwardSlashRegexMode ForwardSlashRegexMode =
+      LexerForwardSlashRegexMode::None;
+
   /// True if we should skip past a `#!` line at the start of the file.
   const bool IsHashbangAllowed;
 
@@ -154,6 +171,19 @@ class Lexer {
 
   void initialize(unsigned Offset, unsigned EndOffset);
 
+  /// Retrieve the diagnostic engine for emitting diagnostics for the current
+  /// token.
+  DiagnosticEngine *getTokenDiags() {
+    return DiagQueue ? &DiagQueue->getDiags() : nullptr;
+  }
+
+  /// Retrieve the underlying diagnostic engine we emit diagnostics to. Note
+  /// this should only be used for diagnostics not concerned with the current
+  /// token.
+  DiagnosticEngine *getUnderlyingDiags() {
+    return DiagQueue ? &DiagQueue->getUnderlyingDiags() : nullptr;
+  }
+
 public:
   /// Create a normal lexer that scans the whole source buffer.
   ///
@@ -209,6 +239,10 @@ class Lexer {
       LeadingTriviaResult = LeadingTrivia;
       TrailingTriviaResult = TrailingTrivia;
     }
+    // Emit any diagnostics recorded for this token.
+    if (DiagQueue)
+      DiagQueue->emit();
+
     if (Result.isNot(tok::eof))
       lexImpl();
   }
@@ -298,12 +332,12 @@ class Lexer {
   void restoreState(State S, bool enableDiagnostics = false) {
     assert(S.isValid());
     CurPtr = getBufferPtrForSourceLoc(S.Loc);
-    // Don't reemit diagnostics while readvancing the lexer.
-    llvm::SaveAndRestore<DiagnosticEngine*>
-      D(Diags, enableDiagnostics ? Diags : nullptr);
-
     lexImpl();
 
+    // Don't re-emit diagnostics from readvancing the lexer.
+    if (DiagQueue && !enableDiagnostics)
+      DiagQueue->clear();
+
     // Restore Trivia.
     if (TriviaRetention == TriviaRetentionMode::WithTrivia)
       LeadingTrivia = S.LeadingTrivia;
@@ -505,7 +539,7 @@ class Lexer {
 
   void getStringLiteralSegments(const Token &Str,
                                 SmallVectorImpl<StringSegment> &Segments) {
-    return getStringLiteralSegments(Str, Segments, Diags);
+    return getStringLiteralSegments(Str, Segments, getTokenDiags());
   }
 
   static SourceLoc getSourceLoc(const char *Loc) {
@@ -531,6 +565,11 @@ class Lexer {
     void operator=(const SILBodyRAII&) = delete;
   };
 
+  /// Attempt to re-lex a regex literal with forward slashes `/.../` from a
+  /// given lexing state. If \p mustBeRegex is set to true, a regex literal will
+  /// always be lexed. Otherwise, it will not be lexed if it may be ambiguous.
+  void tryLexForwardSlashRegexLiteralFrom(State S, bool mustBeRegex);
+
 private:
   /// Nul character meaning kind.
   enum class NulCharacterKind {
@@ -595,8 +634,8 @@ class Lexer {
   void lexStringLiteral(unsigned CustomDelimiterLen = 0);
   void lexEscapedIdentifier();
 
-  /// Attempt to lex a regex literal, returning true if a regex literal was
-  /// lexed, false if this is not a regex literal.
+  /// Attempt to lex a regex literal, returning true if lexing should continue,
+  /// false if this is not a regex literal.
   bool tryLexRegexLiteral(const char *TokStart);
 
   void tryLexEditorPlaceholder();

include/swift/Parse/Parser.h

@@ -559,6 +559,11 @@ class Parser {
     return f(backtrackScope);
   }
 
+  /// Discard the current token. This will avoid interface hashing or updating
+  /// the previous loc. Only should be used if you've completely re-lexed
+  /// a different token at that position.
+  SourceLoc discardToken();
+
   /// Consume a token that we created on the fly to correct the original token
   /// stream from lexer.
   void consumeExtraToken(Token K);
@@ -1752,8 +1757,17 @@ class Parser {
   ParserResult<Expr>
   parseExprPoundCodeCompletion(Optional<StmtKind> ParentKind);
 
+  UnresolvedDeclRefExpr *makeExprOperator(const Token &opToken);
   UnresolvedDeclRefExpr *parseExprOperator();
 
+  /// Try re-lex a '/' operator character as a regex literal. This should be
+  /// called when parsing in an expression position to ensure a regex literal is
+  /// correctly parsed.
+  ///
+  /// If \p mustBeRegex is set to true, a regex literal will always be lexed if
+  /// enabled. Otherwise, it will not be lexed if it may be ambiguous.
+  void tryLexRegexLiteral(bool mustBeRegex);
+
   void validateCollectionElement(ParserResult<Expr> element);
 
   //===--------------------------------------------------------------------===//

lib/AST/DiagnosticEngine.cpp

@@ -1040,24 +1040,40 @@ DiagnosticBehavior DiagnosticState::determineBehavior(const Diagnostic &diag) {
 
 void DiagnosticEngine::flushActiveDiagnostic() {
   assert(ActiveDiagnostic && "No active diagnostic to flush");
+  handleDiagnostic(std::move(*ActiveDiagnostic));
+  ActiveDiagnostic.reset();
+}
+
+void DiagnosticEngine::handleDiagnostic(Diagnostic &&diag) {
   if (TransactionCount == 0) {
-    emitDiagnostic(*ActiveDiagnostic);
+    emitDiagnostic(diag);
     WrappedDiagnostics.clear();
     WrappedDiagnosticArgs.clear();
   } else {
-    onTentativeDiagnosticFlush(*ActiveDiagnostic);
-    TentativeDiagnostics.emplace_back(std::move(*ActiveDiagnostic));
+    onTentativeDiagnosticFlush(diag);
+    TentativeDiagnostics.emplace_back(std::move(diag));
   }
-  ActiveDiagnostic.reset();
+}
+
+void DiagnosticEngine::clearTentativeDiagnostics() {
+  TentativeDiagnostics.clear();
+  WrappedDiagnostics.clear();
+  WrappedDiagnosticArgs.clear();
 }
 
 void DiagnosticEngine::emitTentativeDiagnostics() {
   for (auto &diag : TentativeDiagnostics) {
     emitDiagnostic(diag);
   }
-  TentativeDiagnostics.clear();
-  WrappedDiagnostics.clear();
-  WrappedDiagnosticArgs.clear();
+  clearTentativeDiagnostics();
+}
+
+void DiagnosticEngine::forwardTentativeDiagnosticsTo(
+    DiagnosticEngine &targetEngine) {
+  for (auto &diag : TentativeDiagnostics) {
+    targetEngine.handleDiagnostic(std::move(diag));
+  }
+  clearTentativeDiagnostics();
 }
 
 /// Returns the access level of the least accessible PrettyPrintedDeclarations

lib/Frontend/CompilerInvocation.cpp

@@ -504,6 +504,13 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.EnableExperimentalStringProcessing |=
       Args.hasArg(OPT_enable_experimental_string_processing);
 
+  // Whether '/.../' regex literals are enabled. This implies experimental
+  // string processing.
+  if (Args.hasArg(OPT_enable_bare_slash_regex)) {
+    Opts.EnableBareSlashRegexLiterals = true;
+    Opts.EnableExperimentalStringProcessing = true;
+  }
+
   Opts.EnableExperimentalBoundGenericExtensions |=
     Args.hasArg(OPT_enable_experimental_bound_generic_extensions);
 
@@ -1010,9 +1017,6 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   if (Args.hasArg(OPT_disable_requirement_machine_reuse))
     Opts.EnableRequirementMachineReuse = false;
 
-  if (Args.hasArg(OPT_enable_bare_slash_regex))
-    Opts.EnableForwardSlashRegexLiterals = true;
-
   if (Args.hasArg(OPT_enable_requirement_machine_opaque_archetypes))
     Opts.EnableRequirementMachineOpaqueArchetypes = true;