Parse \P{...}

Handle the following kinds of character properties:
- General category e.g `Lu`, `C`
- Binary properties e.g `Alphabetic`
- Script and script extensions e.g `Greek`
- `Any`, `Assigned` & `Ascii`

This covers the properties required by UTS#18
Level 1, and uses the loose matching specified by
UAX44-LM3, including handling of aliases defined
by Unicode.

This includes a good chunk of the properties
supported by Oniguruma, with the exception of the
`In_...` properties it supports. For now, I've left
them as a separate case that gets parsed without
loose matching, but we may want to eventually fold
them into some of the other cases as aliases. There
are also a couple of PCRE special properties such as
`Xps` that get parsed this way.

For any properties that aren't currently recognized,
they are parsed into an `other` case, as it's
possible we may support them in the future.

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/ASTBuilder.swift

@@ -154,3 +154,8 @@ public func posixSet(
 ) -> Atom {
   return .namedSet(.init(inverted: inverted, set: set))
 }
+func prop(
+  _ kind: Atom.CharacterProperty.Kind, inverted: Bool = false
+) -> Atom {
+  return .property(.init(kind, isInverted: inverted))
+}

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -12,7 +12,7 @@ public enum Atom: Hashable {
   /// A Unicode property, category, or script
   ///
   /// \p{...}, \p{^...}, \P
-  case property(Prop, inverted: Bool)
+  case property(CharacterProperty)
 
   /// A built-in escaped character
   ///
@@ -306,20 +306,51 @@ extension Atom.EscapedBuiltin {
 }
 
 extension Atom {
-  // TODO: Hamish, I believe you have a formulation of this and have
-  // thought through the parsing a whole lot more. This is just what
-  // I have at the time, but let's get something better for the AST
-  // and parser support.
-  public enum Prop: Hashable {
-    case gc(Unicode.GeneralCategory)
-    case pcreSpecial(PCRESpecialCategory)
+  public struct CharacterProperty: Hashable {
+    public var kind: Kind
+    public var isInverted: Bool
+
+    public init(_ kind: Kind, isInverted: Bool) {
+      self.kind = kind
+      self.isInverted = isInverted
+    }
+  }
+}
+
+extension Atom.CharacterProperty {
+  public enum Kind: Hashable {
+    /// Matches any character, equivalent to Oniguruma's '\O'.
+    case any
+
+    // The inverse of 'Unicode.ExtendedGeneralCategory.unassigned'.
+    case assigned
+
+    /// All ascii characters U+00...U+7F
+    case ascii
+
+    /// A general category property.
+    case generalCategory(Unicode.ExtendedGeneralCategory)
+
+    /// Binary character properties. Note that only the following are required
+    /// by UTS#18 Level 1:
+    /// - Alphabetic
+    /// - Uppercase
+    /// - Lowercase
+    /// - White_Space
+    /// - Noncharacter_Code_Point
+    /// - Default_Ignorable_Code_Point
+    case binary(Unicode.BinaryProperty, value: Bool = true)
+
+    /// Character script and script extensions.
     case script(Unicode.Script)
+    case scriptExtension(Unicode.Script)
 
-    // TODO: replace me
-    case propCheck(PropName, PropValue)
+    /// Some special properties implemented by PCRE and Oniguruma.
+    case pcreSpecial(PCRESpecialCategory)
+    case onigurumaSpecial(OnigurumaSpecialProperty)
 
-    // TODO: yuk, let's make sure other cases are a superset of this
-    case oniguruma(FlattendedOnigurumaUnicodeProperty)
+    /// Unhandled properties.
+    case other(key: String?, value: String)
 
     // TODO: erm, separate out or fold into something? splat it in?
     public enum PCRESpecialCategory: String, Hashable {
@@ -329,9 +360,6 @@ extension Atom {
       case universallyNamed = "Xuc"
       case perlWord         = "Xwd"
     }
-
-    public enum PropName: Hashable {}
-    public enum PropValue: Hashable {}
   }
 }
 

Sources/_MatchingEngine/Regex/Parse/CharacterPropertyClassification.swift

@@ -137,6 +137,7 @@ enum ParseError: Error, Hashable {
   case invalidCharacterClassRangeOperand
 
   case invalidPOSIXSetName(String)
+  case emptyProperty
 }
 
 

Sources/_MatchingEngine/Regex/Parse/Diagnostics.swift

@@ -427,6 +427,38 @@ extension Source {
     }
   }
 
+  /// Try to consume a character property.
+  ///
+  ///     Property -> ('p{' | 'P{') Prop ('=' Prop)? '}'
+  ///     Prop -> [\s\w-]+
+  ///
+  private mutating func lexCharacterProperty(
+  ) throws -> Value<Atom.CharacterProperty>? {
+    try recordLoc { src in
+      // '\P{...}' is the inverted version of '\p{...}'
+      guard src.starts(with: "p{") || src.starts(with: "P{") else { return nil }
+      let isInverted = src.peek() == "P"
+      src.eat(count: 2)
+
+      // We should either have:
+      // - '\p{x=y}' where 'x' is a property key, and 'y' is a value.
+      // - '\p{y}' where 'y' is a value (or a bool key with an inferred value
+      //   of true), and its key is inferred.
+      // TODO: We could have better recovery here if we only ate the characters
+      // that property keys and values can use.
+      let lhs = try src.lexUntil({ $0.peek() == "}" || $0.peek() == "=" }).value
+      if src.tryEat("}") {
+        let prop = try Source.classifyCharacterPropertyValueOnly(lhs)
+        return .init(prop, isInverted: isInverted)
+      }
+      src.eat(asserting: "=")
+
+      let rhs = try src.lexUntil(eating: "}").value
+      let prop = try Source.classifyCharacterProperty(key: lhs, value: rhs)
+      return .init(prop, isInverted: isInverted)
+    }
+  }
+
   /// Consume an escaped atom, starting from after the backslash
   ///
   ///     Escaped          -> KeyboardModified | Builtin
@@ -453,6 +485,11 @@ extension Source {
         return .namedCharacter(try src.lexUntil(eating: "}").value)
       }
 
+      // Character property \p{...} \P{...}.
+      if let prop = try src.lexCharacterProperty() {
+        return .property(prop.value)
+      }
+
       let char = src.eat()
 
       // Single-character builtins