Reject literal non-NFC characters

Such characters will have to be spelled using
`\u{...}` syntax instead.

Author: hamishknight

Sources/_RegexParser/Regex/Parse/Diagnostics.swift

@@ -43,6 +43,7 @@ enum ParseError: Error, Hashable {
   case expectedEscape
   case invalidEscape(Character)
   case confusableCharacter(Character)
+  case literalCharMustBeNFC(Character)
 
   case quoteMayNotSpanMultipleLines
   case unsetExtendedSyntaxMayNotSpanMultipleLines
@@ -145,6 +146,8 @@ extension ParseError: CustomStringConvertible {
       return "invalid escape sequence '\\\(c)'"
     case .confusableCharacter(let c):
       return "'\(c)' is confusable for a metacharacter; use '\\u{...}' instead"
+    case .literalCharMustBeNFC(let c):
+      return "'\(c)' may not have the expected scalars; specify them explicitly with '\\u{...}' instead"
     case .quoteMayNotSpanMultipleLines:
       return "quoted sequence may not span multiple lines in multi-line literal"
     case .unsetExtendedSyntaxMayNotSpanMultipleLines:

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -2055,13 +2055,14 @@ extension Parser {
         return .invalid
       }
 
-      guard let charLoc = p.tryEatWithLoc() else {
+      guard let charWithLoc = p.tryEatWithLoc() else {
         // We check at the beginning of the function for `isEmpty`, so we should
         // not be at the end of the input here.
         p.unreachable("Unexpected end of input")
         return nil
       }
-      let char = charLoc.value
+      let char = charWithLoc.value
+      let charLoc = charWithLoc.location
       switch char {
       case ")", "|":
         if customCC {
@@ -2092,7 +2093,11 @@ extension Parser {
         let scalars = char.unicodeScalars
         if scalars.count > 1 && scalars.first!.isASCII && char != "\r\n" &&
             !char.isLetter && !char.isNumber {
-          p.error(.confusableCharacter(char), at: charLoc.location)
+          p.error(.confusableCharacter(char), at: charLoc)
+        }
+        // Reject unescaped non-NFC characters.
+        if !char.isNFC {
+          p.error(.literalCharMustBeNFC(char), at: charLoc)
         }
         break
       }

Tests/RegexTests/CompileTests.swift

@@ -332,7 +332,7 @@ extension RegexTests {
       doesNotContain: [.match, .consumeBy, .matchScalarUnchecked])
     // quoted literal is not all ascii -> match scalar when possible, always do boundary checks
     expectProgram(
-      for: "aaa\u{301}",
+      for: #"aaa\u{301}"#,
       contains: [.match, .matchScalar],
       doesNotContain: [.consumeBy, .matchScalarUnchecked])
     // scalar mode -> always emit match scalar without boundary checks
@@ -347,7 +347,7 @@ extension RegexTests {
       contains: [.matchScalarUnchecked],
       doesNotContain: [.match, .consumeBy, .matchScalar])
     expectProgram(
-      for: "aaa\u{301}",
+      for: #"aaa\u{301}"#,
       semanticLevel: .unicodeScalar,
       contains: [.matchScalarUnchecked],
       doesNotContain: [.match, .consumeBy, .matchScalar])

Tests/RegexTests/MatchTests.swift

@@ -226,7 +226,7 @@ extension RegexTests {
 
     // MARK: Allowed combining characters
 
-    firstMatchTest("e\u{301}", input: "e\u{301}", match: "e\u{301}")
+    firstMatchTest(#"e\u{301}"#, input: "e\u{301}", match: "e\u{301}")
     firstMatchTest("1\u{358}", input: "1\u{358}", match: "1\u{358}")
     firstMatchTest(#"\ \#u{361}"#, input: " \u{361}", match: " \u{361}")
 
@@ -774,7 +774,7 @@ extension RegexTests {
     firstMatchTest(#"[\d]"#, input: "1️⃣", match: "1️⃣")
     firstMatchTest(#"(?P)[\d]"#, input: "1️⃣", match: nil)
     firstMatchTest("[0-2&&1-3]", input: "1️⃣", match: nil)
-    firstMatchTest("[1-2e\u{301}]", input: "1️⃣", match: nil)
+    firstMatchTest(#"[1-2e\u{301}]"#, input: "1️⃣", match: nil)
 
     firstMatchTest(#"[\u{3A9}-\u{3A9}]"#, input: "\u{3A9}", match: "\u{3A9}")
 

Tests/RegexTests/ParseTests.swift

@@ -374,10 +374,27 @@ extension RegexTests {
 
     // MARK: Allowed combining characters
 
-    parseTest("e\u{301}", "e\u{301}")
     parseTest("1\u{358}", "1\u{358}")
     parseTest(#"\ \#u{361}"#, " \u{361}")
 
+    // We don't allow non-NFC literal characters.
+    parseTest(
+      "e\u{301}", "e\u{301}", throwsError: .literalCharMustBeNFC("e\u{301}")
+    )
+    parseTest(
+      "[e\u{301}]", charClass("e\u{301}"),
+      throwsError: .literalCharMustBeNFC("e\u{301}")
+    )
+    // Can't be escaped either, must be written using `\u{...}`.
+    parseTest(
+      "\\e\u{301}", "e\u{301}", throwsError: .invalidEscape("e\u{301}"))
+    parseTest(
+      "[\\e\u{301}]", charClass("e\u{301}"),
+      throwsError: .invalidEscape("e\u{301}")
+    )
+    parseTest("\u{E9}", "e\u{301}")
+    parseTest("[\u{E9}]", charClass("e\u{301}"))
+
     // MARK: Alternations
 
     parseTest(
@@ -2889,7 +2906,10 @@ extension RegexTests {
     diagnosticTest(#"|([👨‍👩‍👦-👩‍👩‍👧‍👧])?"#, .invalidCharacterClassRangeOperand)
 
     // Not single-scalar NFC.
-    diagnosticTest("[e\u{301}-e\u{302}]", .invalidCharacterClassRangeOperand)
+    diagnosticTest(
+      "[e\u{301}-e\u{302}]", .invalidCharacterClassRangeOperand,
+      .literalCharMustBeNFC("e\u{301}"), .literalCharMustBeNFC("e\u{302}")
+    )
 
     // These scalar values expand under NFC.
     let nfcExpandingScalars: [UInt32] = [