Allow certain escape sequences in character class ranges

Certain escape sequences express a unicode scalar
and as such are valid in a range.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -641,17 +641,67 @@ extension AST.Atom {
     case .scalar(let s):
       return Character(s)
 
+    case .escaped(let c):
+      switch c {
+      // TODO: Should we separate these into a separate enum? Or move the
+      // specifics of the scalar to the DSL tree?
+      case .alarm:
+        return "\u{7}"
+      case .backspace:
+        return "\u{8}"
+      case .escape:
+        return "\u{1B}"
+      case .formfeed:
+        return "\u{C}"
+      case .newline:
+        return "\n"
+      case .carriageReturn:
+        return "\r"
+      case .tab:
+        return "\t"
+
+      case .singleDataUnit, .decimalDigit, .notDecimalDigit,
+          .horizontalWhitespace, .notHorizontalWhitespace, .notNewline,
+          .newlineSequence, .whitespace, .notWhitespace, .verticalTab,
+          .notVerticalTab, .wordCharacter, .notWordCharacter, .graphemeCluster,
+          .wordBoundary, .notWordBoundary, .startOfSubject,
+          .endOfSubjectBeforeNewline, .endOfSubject,
+          .firstMatchingPositionInSubject, .resetStartOfMatch, .trueAnychar,
+          .textSegment, .notTextSegment:
+        return nil
+      }
+
     case .keyboardControl, .keyboardMeta, .keyboardMetaControl:
-      // TODO: Not a character per-say, what should we do?
-      fallthrough
+      // TODO: These should have unicode scalar values.
+      return nil
 
-    case .property, .escaped, .any, .startOfLine, .endOfLine,
-        .backreference, .subpattern, .namedCharacter, .callout,
-        .backtrackingDirective:
+    case .namedCharacter:
+      // TODO: This should have a unicode scalar value depending on the name
+      // given.
+      // TODO: Do we want to validate and assign a scalar value when building
+      // the AST? Or defer for the matching engine?
+      return nil
+
+    case .property, .any, .startOfLine, .endOfLine, .backreference, .subpattern,
+        .callout, .backtrackingDirective:
       return nil
     }
   }
 
+  /// Whether this atom is valid as the operand of a custom character class
+  /// range.
+  public var isValidCharacterClassRangeBound: Bool {
+    // If we have a literal character value for this, it can be used as a bound.
+    if literalCharacterValue != nil { return true }
+    switch kind {
+    // \cx, \C-x, \M-x, \M-\C-x, \N{...}
+    case .keyboardControl, .keyboardMeta, .keyboardMetaControl, .namedCharacter:
+      return true
+    default:
+      return false
+    }
+  }
+
   /// Produce a string literal representation of the atom, if possible
   ///
   /// Individual characters will be returned, Unicode scalars will be

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -498,10 +498,11 @@ extension Parser {
       // Range between atoms.
       if let (dashLoc, rhs) =
           try source.lexCustomCharClassRangeEnd(context: context) {
-        guard atom.literalCharacterValue != nil &&
-              rhs.literalCharacterValue != nil else {
+        guard atom.isValidCharacterClassRangeBound &&
+              rhs.isValidCharacterClassRangeBound else {
           throw ParseError.invalidCharacterClassRangeOperand
         }
+        // TODO: Validate lower <= upper?
         members.append(.range(.init(atom, dashLoc, rhs)))
         continue
       }

Tests/RegexTests/MatchTests.swift

@@ -585,6 +585,35 @@ extension RegexTests {
 
     firstMatchTest("[[:script=Greek:]]", input: "123αβγxyz", match: "α")
 
+    func scalar(_ u: UnicodeScalar) -> UInt32 { u.value }
+
+    // Currently not supported in the matching engine.
+    for s in scalar("\u{C}") ... scalar("\u{1B}") {
+      let u = UnicodeScalar(s)!
+      firstMatchTest(#"[\f-\e]"#, input: "\u{B}\u{1C}\(u)", match: "\(u)",
+                     xfail: true)
+    }
+    for u: UnicodeScalar in ["\u{7}", "\u{8}"] {
+      firstMatchTest(#"[\a-\b]"#, input: "\u{6}\u{9}\(u)", match: "\(u)",
+                     xfail: true)
+    }
+    for s in scalar("\u{A}") ... scalar("\u{D}") {
+      let u = UnicodeScalar(s)!
+      firstMatchTest(#"[\n-\r]"#, input: "\u{9}\u{E}\(u)", match: "\(u)",
+                     xfail: true)
+    }
+    firstMatchTest(#"[\t-\t]"#, input: "\u{8}\u{A}\u{9}", match: "\u{9}",
+                   xfail: true)
+
+    for c: UnicodeScalar in ["a", "b", "c"] {
+      firstMatchTest(#"[\c!-\C-#]"#, input: "def\(c)", match: "\(c)",
+                     xfail: true)
+    }
+    for c: UnicodeScalar in ["$", "%", "&", "'"] {
+      firstMatchTest(#"[\N{DOLLAR SIGN}-\N{APOSTROPHE}]"#,
+                     input: "#()\(c)", match: "\(c)", xfail: true)
+    }
+
     // MARK: Operators
 
     firstMatchTest(

Tests/RegexTests/ParseTests.swift

@@ -494,6 +494,25 @@ extension RegexTests {
     parseTest("[*]", charClass("*"))
     parseTest("[{0}]", charClass("{", "0", "}"))
 
+    parseTest(#"[\f-\e]"#, charClass(
+      range_m(.escaped(.formfeed), .escaped(.escape))))
+    parseTest(#"[\a-\b]"#, charClass(
+      range_m(.escaped(.alarm), .escaped(.backspace))))
+    parseTest(#"[\n-\r]"#, charClass(
+      range_m(.escaped(.newline), .escaped(.carriageReturn))))
+    parseTest(#"[\t-\t]"#, charClass(
+      range_m(.escaped(.tab), .escaped(.tab))))
+
+    parseTest(#"[\cX-\cY\C-A-\C-B\M-\C-A-\M-\C-B\M-A-\M-B]"#, charClass(
+      range_m(.keyboardControl("X"), .keyboardControl("Y")),
+      range_m(.keyboardControl("A"), .keyboardControl("B")),
+      range_m(.keyboardMetaControl("A"), .keyboardMetaControl("B")),
+      range_m(.keyboardMeta("A"), .keyboardMeta("B"))
+    ))
+
+    parseTest(#"[\N{DOLLAR SIGN}-\N{APOSTROPHE}]"#, charClass(
+      range_m(.namedCharacter("DOLLAR SIGN"), .namedCharacter("APOSTROPHE"))))
+
     // MARK: Operators
 
     parseTest(
@@ -575,6 +594,15 @@ extension RegexTests {
     // Escaped U+3000 IDEOGRAPHIC SPACE.
     parseTest(#"\\#u{3000}"#, "\u{3000}")
 
+    // Control and meta controls.
+    parseTest(#"\c "#, atom(.keyboardControl(" ")))
+    parseTest(#"\c!"#, atom(.keyboardControl("!")))
+    parseTest(#"\c~"#, atom(.keyboardControl("~")))
+    parseTest(#"\C--"#, atom(.keyboardControl("-")))
+    parseTest(#"\M-\C-a"#, atom(.keyboardMetaControl("a")))
+    parseTest(#"\M-\C--"#, atom(.keyboardMetaControl("-")))
+    parseTest(#"\M-a"#, atom(.keyboardMeta("a")))
+
     // MARK: Comments
 
     parseTest(
@@ -1877,6 +1905,9 @@ extension RegexTests {
 
     diagnosticTest("\\", .expectedEscape)
 
+    // TODO: Custom diagnostic for control sequence
+    diagnosticTest(#"\c"#, .unexpectedEndOfInput)
+
     // TODO: Custom diagnostic for expected backref
     diagnosticTest(#"\g"#, .invalidEscape("g"))
     diagnosticTest(#"\k"#, .invalidEscape("k"))