Merge pull request #58 from hamishknight/lotta-props

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/ASTBuilder.swift

@@ -152,5 +152,10 @@ public func charClass(
 public func posixSet(
   _ set: Unicode.POSIXCharacterSet, inverted: Bool = false
 ) -> Atom {
-  return .named(.init(inverted: inverted, set: set))
+  return .namedSet(.init(inverted: inverted, set: set))
+}
+func prop(
+  _ kind: Atom.CharacterProperty.Kind, inverted: Bool = false
+) -> Atom {
+  return .property(.init(kind, isInverted: inverted))
 }

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -12,7 +12,7 @@ public enum Atom: Hashable {
   /// A Unicode property, category, or script
   ///
   /// \p{...}, \p{^...}, \P
-  case property(Prop, inverted: Bool)
+  case property(CharacterProperty)
 
   /// A built-in escaped character
   ///
@@ -31,7 +31,10 @@ public enum Atom: Hashable {
   /// A named set (using POSIX syntax)
   ///
   /// [:...:], [:^...:]
-  case named(POSIXSet)
+  case namedSet(POSIXSet)
+
+  /// A named character \N{...}
+  case namedCharacter(String)
 
   /// .
   case any
@@ -303,20 +306,51 @@ extension Atom.EscapedBuiltin {
 }
 
 extension Atom {
-  // TODO: Hamish, I believe you have a formulation of this and have
-  // thought through the parsing a whole lot more. This is just what
-  // I have at the time, but let's get something better for the AST
-  // and parser support.
-  public enum Prop: Hashable {
-    case gc(Unicode.GeneralCategory)
-    case pcreSpecial(PCRESpecialCategory)
+  public struct CharacterProperty: Hashable {
+    public var kind: Kind
+    public var isInverted: Bool
+
+    public init(_ kind: Kind, isInverted: Bool) {
+      self.kind = kind
+      self.isInverted = isInverted
+    }
+  }
+}
+
+extension Atom.CharacterProperty {
+  public enum Kind: Hashable {
+    /// Matches any character, equivalent to Oniguruma's '\O'.
+    case any
+
+    // The inverse of 'Unicode.ExtendedGeneralCategory.unassigned'.
+    case assigned
+
+    /// All ascii characters U+00...U+7F
+    case ascii
+
+    /// A general category property.
+    case generalCategory(Unicode.ExtendedGeneralCategory)
+
+    /// Binary character properties. Note that only the following are required
+    /// by UTS#18 Level 1:
+    /// - Alphabetic
+    /// - Uppercase
+    /// - Lowercase
+    /// - White_Space
+    /// - Noncharacter_Code_Point
+    /// - Default_Ignorable_Code_Point
+    case binary(Unicode.BinaryProperty, value: Bool = true)
+
+    /// Character script and script extensions.
     case script(Unicode.Script)
+    case scriptExtension(Unicode.Script)
 
-    // TODO: replace me
-    case propCheck(PropName, PropValue)
+    /// Some special properties implemented by PCRE and Oniguruma.
+    case pcreSpecial(PCRESpecialCategory)
+    case onigurumaSpecial(OnigurumaSpecialProperty)
 
-    // TODO: yuk, let's make sure other cases are a superset of this
-    case oniguruma(FlattendedOnigurumaUnicodeProperty)
+    /// Unhandled properties.
+    case other(key: String?, value: String)
 
     // TODO: erm, separate out or fold into something? splat it in?
     public enum PCRESpecialCategory: String, Hashable {
@@ -326,9 +360,6 @@ extension Atom {
       case universallyNamed = "Xuc"
       case perlWord         = "Xwd"
     }
-
-    public enum PropName: Hashable {}
-    public enum PropValue: Hashable {}
   }
 }
 
@@ -372,8 +403,10 @@ extension Atom: _ASTPrintable {
       fatalError("TODO")
     case .keyboardMetaControl(_):
       fatalError("TODO")
-    case .named:
+    case .namedSet:
       fatalError("TODO")
+    case .namedCharacter(let charName):
+      return "\\N{\(charName)}"
     case .any: return "."
     case .startOfLine: return "^"
     case .endOfLine: return "$"
@@ -407,8 +440,8 @@ extension Atom {
     case .keyboardControl, .keyboardMeta, .keyboardMetaControl:
       fatalError("TODO")
 
-    case .property, .escaped, .named, .any, .startOfLine, .endOfLine,
-        .backreference, .subpattern, .condition, .trivia:
+    case .property, .escaped, .namedSet, .any, .startOfLine, .endOfLine,
+        .backreference, .subpattern, .condition, .trivia, .namedCharacter:
       return nil
     }
   }

Sources/_MatchingEngine/Regex/Parse/CharacterPropertyClassification.swift

@@ -137,6 +137,7 @@ enum ParseError: Error, Hashable {
   case invalidCharacterClassRangeOperand
 
   case invalidPOSIXSetName(String)
+  case emptyProperty
 }
 
 

Sources/_MatchingEngine/Regex/Parse/Diagnostics.swift

@@ -230,24 +230,32 @@ extension Source {
   }
 
   private mutating func lexUntil(
-    _ end: String, validate: (String) throws -> Void = { _ in }
+    _ predicate: (inout Source) -> Bool,
+    validate: (String) throws -> Void = { _ in }
   ) throws -> Value<String> {
     try recordLoc { src in
       var result = ""
-      while !src.tryEat(sequence: end) {
+      while !predicate(&src) {
         // TODO(diagnostic): expected `end`, instead of end-of-input
 
         result.append(src.eat())
       }
+      try validate(result)
       return result
     }
   }
 
+  private mutating func lexUntil(
+    eating end: String, validate: (String) throws -> Void = { _ in }
+  ) throws -> Value<String> {
+    try lexUntil({ src in src.tryEat(sequence: end) }, validate: validate)
+  }
+
   /// Expect a linear run of non-nested non-empty content
   private mutating func expectQuoted(
     endingWith end: String
   ) throws -> Value<String> {
-    try lexUntil(end, validate: { result in
+    try lexUntil(eating: end, validate: { result in
       guard !result.isEmpty else {
         throw ParseError.misc("Expected non-empty contents")
       }
@@ -305,7 +313,8 @@ extension Source {
   ///
   /// Does nothing unless `SyntaxOptions.nonSemanticWhitespace` is set
   mutating func lexNonSemanticWhitespace() throws -> Value<()>? {
-    try recordLoc { src in
+    guard syntax.ignoreWhitespace else { return nil }
+    return try recordLoc { src in
       var didSomething = false
       while src.tryEat(" ") {
         didSomething = true
@@ -410,14 +419,46 @@ extension Source {
     try recordLoc { src in
       guard src.tryEat(sequence: "[:") else { return nil }
       let inverted = src.tryEat("^")
-      let name = try src.lexUntil(":]").value
+      let name = try src.lexUntil(eating: ":]").value
       guard let set = Unicode.POSIXCharacterSet(rawValue: name) else {
         throw ParseError.invalidPOSIXSetName(name)
       }
       return Atom.POSIXSet(inverted: inverted, set: set)
     }
   }
 
+  /// Try to consume a character property.
+  ///
+  ///     Property -> ('p{' | 'P{') Prop ('=' Prop)? '}'
+  ///     Prop -> [\s\w-]+
+  ///
+  private mutating func lexCharacterProperty(
+  ) throws -> Value<Atom.CharacterProperty>? {
+    try recordLoc { src in
+      // '\P{...}' is the inverted version of '\p{...}'
+      guard src.starts(with: "p{") || src.starts(with: "P{") else { return nil }
+      let isInverted = src.peek() == "P"
+      src.eat(count: 2)
+
+      // We should either have:
+      // - '\p{x=y}' where 'x' is a property key, and 'y' is a value.
+      // - '\p{y}' where 'y' is a value (or a bool key with an inferred value
+      //   of true), and its key is inferred.
+      // TODO: We could have better recovery here if we only ate the characters
+      // that property keys and values can use.
+      let lhs = try src.lexUntil({ $0.peek() == "}" || $0.peek() == "=" }).value
+      if src.tryEat("}") {
+        let prop = try Source.classifyCharacterPropertyValueOnly(lhs)
+        return .init(prop, isInverted: isInverted)
+      }
+      src.eat(asserting: "=")
+
+      let rhs = try src.lexUntil(eating: "}").value
+      let prop = try Source.classifyCharacterProperty(key: lhs, value: rhs)
+      return .init(prop, isInverted: isInverted)
+    }
+  }
+
   /// Consume an escaped atom, starting from after the backslash
   ///
   ///     Escaped          -> KeyboardModified | Builtin
@@ -439,6 +480,16 @@ extension Source {
         return .keyboardMeta(try src.expectASCII().value)
       }
 
+      // Named character \N{...}
+      if src.tryEat(sequence: "N{") {
+        return .namedCharacter(try src.lexUntil(eating: "}").value)
+      }
+
+      // Character property \p{...} \P{...}.
+      if let prop = try src.lexCharacterProperty() {
+        return .property(prop.value)
+      }
+
       let char = src.eat()
 
       // Single-character builtins
@@ -491,7 +542,7 @@ extension Source {
       // TODO: Can we try and recover and diagnose for named sets outside
       // character classes?
       if customCC, let set = try src.lexPOSIXNamedSet()?.value {
-        return .named(set)
+        return .namedSet(set)
       }
 
       let char = src.eat()