Introduce scalar sequences \u{AA BB CC}

Allow a whitespace-separated list of scalars within
the `\u{...}` syntax. This is syntactic sugar that
gets implicitly splatted out, for example `\u{A B C}`
becomes `\u{A}\u{B}\u{C}`.

Author: hamishknight

Sources/_RegexParser/Regex/AST/Atom.swift

@@ -31,6 +31,12 @@ extension AST {
       /// \u{...}, \0dd, \x{...}, ...
       case scalar(Scalar)
 
+      /// A whitespace-separated sequence of Unicode scalar values which are
+      /// implicitly splatted out.
+      ///
+      /// `\u{A B C}` -> `\u{A}\u{B}\u{C}`
+      case scalarSequence(ScalarSequence)
+
       /// A Unicode property, category, or script, including those written using
       /// POSIX syntax.
       ///
@@ -84,6 +90,7 @@ extension AST.Atom {
     switch kind {
     case .char(let v):                  return v
     case .scalar(let v):                return v
+    case .scalarSequence(let v):        return v
     case .property(let v):              return v
     case .escaped(let v):               return v
     case .keyboardControl(let v):       return v
@@ -116,6 +123,18 @@ extension AST.Atom {
       self.location = location
     }
   }
+
+  public struct ScalarSequence: Hashable {
+    public var scalars: [Scalar]
+    public var trivia: [AST.Trivia]
+
+    public init(_ scalars: [Scalar], trivia: [AST.Trivia]) {
+      precondition(scalars.count > 1, "Expected multiple scalars")
+      self.scalars = scalars
+      self.trivia = trivia
+    }
+    public var scalarValues: [Unicode.Scalar] { scalars.map(\.value) }
+  }
 }
 
 extension AST.Atom {
@@ -708,8 +727,9 @@ extension AST.Atom {
       // the AST? Or defer for the matching engine?
       return nil
 
-    case .property, .any, .startOfLine, .endOfLine, .backreference, .subpattern,
-        .callout, .backtrackingDirective, .changeMatchingOptions:
+    case .scalarSequence, .property, .any, .startOfLine, .endOfLine,
+        .backreference, .subpattern, .callout, .backtrackingDirective,
+        .changeMatchingOptions:
       return nil
     }
   }
@@ -723,6 +743,8 @@ extension AST.Atom {
     // \cx, \C-x, \M-x, \M-\C-x, \N{...}
     case .keyboardControl, .keyboardMeta, .keyboardMetaControl, .namedCharacter:
       return true
+    case .scalarSequence:
+      return true
     default:
       return false
     }
@@ -731,13 +753,21 @@ extension AST.Atom {
   /// A string literal representation of the atom, if possible.
   ///
   /// Individual characters are returned as-is, and Unicode scalars are
-  /// presented using "\u{nnnn}" syntax.
+  /// presented using "\u{nn nn ...}" syntax.
   public var literalStringValue: String? {
+    func scalarLiteral(_ u: [UnicodeScalar]) -> String {
+      let digits = u.map { String($0.value, radix: 16, uppercase: true) }
+        .joined(separator: " ")
+      return "\\u{\(digits)}"
+    }
     switch kind {
     case .char(let c):
       return String(c)
     case .scalar(let s):
-      return "\\u{\(String(s.value.value, radix: 16, uppercase: true))}"
+      return scalarLiteral([s.value])
+
+    case .scalarSequence(let s):
+      return scalarLiteral(s.scalarValues)
 
     case .keyboardControl(let x):
       return "\\C-\(x)"

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -290,10 +290,54 @@ extension Source {
     return try Source.validateUnicodeScalar(str, .hex)
   }
 
+  /// Try to lex a seqence of hex digit unicode scalars.
+  ///
+  ///     UniScalarSequence   -> Whitespace? UniScalarSequencElt+
+  ///     UniScalarSequencElt -> HexDigit{1...} Whitespace?
+  ///
+  mutating func expectUnicodeScalarSequence(
+    eating ending: Character
+  ) throws -> AST.Atom.Kind {
+    try recordLoc { src in
+      var scalars = [AST.Atom.Scalar]()
+      var trivia = [AST.Trivia]()
+
+      // Eat up any leading whitespace.
+      if let t = src.lexWhitespace() { trivia.append(t) }
+
+      while true {
+        let str = src.lexUntil { src in
+          // Hit the ending, stop lexing.
+          if src.isEmpty || src.peek() == ending {
+            return true
+          }
+          // Eat up trailing whitespace, and stop lexing to record the scalar.
+          if let t = src.lexWhitespace() {
+            trivia.append(t)
+            return true
+          }
+          // Not the ending or trivia, must be a digit of the scalar.
+          return false
+        }
+        guard !str.value.isEmpty else { break }
+        scalars.append(try Source.validateUnicodeScalar(str, .hex))
+      }
+      guard !scalars.isEmpty else {
+        throw ParseError.expectedNumber("", kind: .hex)
+      }
+      try src.expect(ending)
+
+      if scalars.count == 1 {
+        return .scalar(scalars[0])
+      }
+      return .scalarSequence(.init(scalars, trivia: trivia))
+    }.value
+  }
+
   /// Eat a scalar off the front, starting from after the
   /// backslash and base character (e.g. `\u` or `\x`).
   ///
-  ///     UniScalar -> 'u{' HexDigit{1...} '}'
+  ///     UniScalar -> 'u{' UniScalarSequence '}'
   ///                | 'u'  HexDigit{4}
   ///                | 'x{' HexDigit{1...} '}'
   ///                | 'x'  HexDigit{0...2}
@@ -314,7 +358,10 @@ extension Source {
       // TODO: PCRE offers a different behavior if PCRE2_ALT_BSUX is set.
       switch base {
       // Hex numbers.
-      case "u" where src.tryEat("{"), "x" where src.tryEat("{"):
+      case "u" where src.tryEat("{"):
+        return try src.expectUnicodeScalarSequence(eating: "}")
+
+      case "x" where src.tryEat("{"):
         let str = try src.lexUntil(eating: "}")
         return .scalar(try Source.validateUnicodeScalar(str, .hex))
 
@@ -598,6 +645,16 @@ extension Source {
     // inside a custom character class (and only treats whitespace as
     // non-semantic there for the extra-extended `(?xx)` mode). If we get a
     // strict-PCRE mode, we'll need to add a case for that.
+    return lexWhitespace()
+  }
+
+  /// Try to consume whitespace as trivia
+  ///
+  ///     Whitespace -> WhitespaceChar+
+  ///
+  /// Unlike `lexNonSemanticWhitespace`, this will always attempt to lex
+  /// whitespace.
+  mutating func lexWhitespace() -> AST.Trivia? {
     let trivia: Located<String>? = recordLoc { src in
       src.tryEatPrefix(\.isPatternWhitespace)?.string
     }

Sources/_RegexParser/Regex/Printing/DumpAST.swift

@@ -138,6 +138,9 @@ extension AST.Atom {
     switch kind {
     case .escaped(let c): return "\\\(c.character)"
 
+    case .scalarSequence(let s):
+      return s.scalars.map(\.value.halfWidthCornerQuoted).joined()
+
     case .namedCharacter(let charName):
       return "\\N{\(charName)}"
 

Sources/_StringProcessing/ConsumerInterface.swift

@@ -230,9 +230,9 @@ extension AST.Atom {
       // handled in emitAssertion
       return nil
 
-    case .escaped, .keyboardControl, .keyboardMeta, .keyboardMetaControl,
-        .backreference, .subpattern, .callout, .backtrackingDirective,
-        .changeMatchingOptions:
+    case .scalarSequence, .escaped, .keyboardControl, .keyboardMeta,
+        .keyboardMetaControl, .backreference, .subpattern, .callout,
+        .backtrackingDirective, .changeMatchingOptions:
       // FIXME: implement
       return nil
     }

Sources/_StringProcessing/PrintAsPattern.swift

@@ -671,13 +671,19 @@ extension AST.Atom {
   }
 
   var _dslBase: String {
+    func scalarLiteral(_ s: UnicodeScalar) -> String {
+      let hex = String(s.value, radix: 16, uppercase: true)
+      return "\\u{\(hex)}"
+    }
     switch kind {
     case let .char(c):
       return String(c)
 
     case let .scalar(s):
-      let hex = String(s.value.value, radix: 16, uppercase: true)
-      return "\\u{\(hex)}"
+      return scalarLiteral(s.value)
+
+    case let .scalarSequence(seq):
+      return seq.scalarValues.map(scalarLiteral).joined()
 
     case let .property(p):
       return p._dslBase
@@ -769,13 +775,9 @@ extension AST.Atom {
 
   var _regexBase: String {
     switch kind {
-    case let .char(c):
-      return String(c)
-      
-    case let .scalar(s):
-      let hex = String(s.value.value, radix: 16, uppercase: true)
-      return "\\u{\(hex)}"
-      
+    case .char, .scalar, .scalarSequence:
+      return literalStringValue!
+
     case let .property(p):
       return p._regexBase
 

Sources/_StringProcessing/Regex/ASTConversion.swift

@@ -60,15 +60,17 @@ extension AST.Node {
           var result = ""
           var idx = idx
           while idx < astChildren.endIndex {
-            let atom: AST.Atom? = astChildren[idx].as()
+            guard let atom: AST.Atom = astChildren[idx].as() else { break }
 
             // TODO: For printing, nice to coalesce
             // scalars literals too. We likely need a different
             // approach even before we have a better IR.
-            if let char = atom?.singleCharacter  {
+            if let char = atom.singleCharacter  {
               result.append(char)
-            } else if let scalar = atom?.singleScalar {
+            } else if let scalar = atom.singleScalar {
               result.append(Character(scalar))
+            } else if case .scalarSequence(let seq) = atom.kind {
+              result += seq.scalarValues.map(Character.init)
             } else {
               break
             }
@@ -136,7 +138,15 @@ extension AST.Node {
         return .trivia(v.contents)
 
       case let .atom(v):
-        return .atom(v.dslTreeAtom)
+        switch v.kind {
+        case .scalarSequence(let seq):
+          // Scalar sequences are splatted into concatenated scalars, which
+          // becomes a quoted literal. Sequences nested in concatenations have
+          // already been coalesced, this just handles the lone atom case.
+          return .quotedLiteral(String(seq.scalarValues.map(Character.init)))
+        default:
+          return .atom(v.dslTreeAtom)
+        }
 
       case let .customCharacterClass(ccc):
         return .customCharacterClass(ccc.dslTreeClass)
@@ -158,47 +168,68 @@ extension AST.Node {
 
 extension AST.CustomCharacterClass {
   var dslTreeClass: DSLTree.CustomCharacterClass {
+    // Convert an atom of the AST. Due to scalar sequences, this may be
+    // converted into multiple DSL atoms.
+    func convertAtom(_ atom: AST.Atom) -> [DSLTree.Atom] {
+      switch atom.kind {
+      case .scalarSequence(let seq):
+        // Scalar sequences are splatted into scalar members.
+        return seq.scalarValues.map { .scalar($0) }
+      default:
+        return [atom.dslTreeAtom]
+      }
+    }
+
     // TODO: Not quite 1-1
     func convert(
       _ member: Member
-    ) -> DSLTree.CustomCharacterClass.Member {
+    ) -> [DSLTree.CustomCharacterClass.Member] {
       switch member {
       case let .custom(ccc):
-        return .custom(ccc.dslTreeClass)
+        return [.custom(ccc.dslTreeClass)]
 
       case let .range(r):
-        return .range(
-          r.lhs.dslTreeAtom, r.rhs.dslTreeAtom)
+        let lhs = convertAtom(r.lhs)
+        let rhs = convertAtom(r.rhs)
+
+        // The LHS and RHS may have been splatted into individual atoms. The
+        // range only covers the last atom of the LHS, and the first atom of
+        // the RHS.
+        var result = [DSLTree.CustomCharacterClass.Member]()
+        result += lhs.dropLast().map { .atom($0) }
+        result.append(.range(lhs.last!, rhs.first!))
+        result += rhs.dropFirst().map { .atom($0) }
+        return result
 
       case let .atom(a):
-        return .atom(a.dslTreeAtom)
+        return convertAtom(a).map { .atom($0) }
 
       case let .quote(q):
-        return .quotedLiteral(q.literal)
+        return [.quotedLiteral(q.literal)]
 
       case let .setOperation(lhs, op, rhs):
         let lhs = DSLTree.CustomCharacterClass(
-          members: lhs.map(convert),
+          members: lhs.flatMap(convert),
           isInverted: false)
         let rhs = DSLTree.CustomCharacterClass(
-          members: rhs.map(convert),
+          members: rhs.flatMap(convert),
           isInverted: false)
 
         switch op.value {
         case .subtraction:
-          return .subtraction(lhs, rhs)
+          return [.subtraction(lhs, rhs)]
         case .intersection:
-          return .intersection(lhs, rhs)
+          return [.intersection(lhs, rhs)]
         case .symmetricDifference:
-          return .symmetricDifference(lhs, rhs)
+          return [.symmetricDifference(lhs, rhs)]
         }
       case let .trivia(t):
-        return .trivia(t.contents)
+        return [.trivia(t.contents)]
       }
     }
 
     return .init(
-      members: members.map(convert),
+      members: members.flatMap(convert),
       isInverted: self.isInverted)
   }
 }

Sources/_StringProcessing/Utility/ASTBuilder.swift

@@ -338,10 +338,26 @@ func escaped(
   atom(.escaped(e))
 }
 func scalar(_ s: Unicode.Scalar) -> AST.Node {
-  atom(.scalar(.init(s, .fake)))
+  .atom(scalar_a(s))
+}
+func scalar_a(_ s: Unicode.Scalar) -> AST.Atom {
+  atom_a(.scalar(.init(s, .fake)))
 }
 func scalar_m(_ s: Unicode.Scalar) -> AST.CustomCharacterClass.Member {
-  atom_m(.scalar(.init(s, .fake)))
+  .atom(scalar_a(s))
+}
+
+func scalarSeq(_ s: Unicode.Scalar...) -> AST.Node {
+  .atom(scalarSeq_a(s))
+}
+func scalarSeq_a(_ s: Unicode.Scalar...) -> AST.Atom {
+  scalarSeq_a(s)
+}
+func scalarSeq_a(_ s: [Unicode.Scalar]) -> AST.Atom {
+  atom_a(.scalarSequence(.init(s.map { .init($0, .fake) }, trivia: [])))
+}
+func scalarSeq_m(_ s: Unicode.Scalar...) -> AST.CustomCharacterClass.Member {
+  .atom(scalarSeq_a(s))
 }
 
 func backreference(_ r: AST.Reference.Kind, recursionLevel: Int? = nil) -> AST.Node {