Custom regex components support (#163)

Hook up custom matchers.

This is only lightly-tested, more exhaustive testing TBD.

Author: milseman

Sources/_MatchingEngine/Regex/Parse/CaptureStructure.swift

@@ -93,6 +93,23 @@ extension CaptureStructure.Constructor {
     }
   }
 
+  public mutating func grouping<T: _TreeNode>(
+    _ child: T,
+    as kind: AST.Group.Kind,
+    withType type: AnyType
+  ) -> CaptureStructure {
+    let innerCaptures = child._captureStructure(&self)
+    switch kind {
+    case .capture:
+      return .atom(type: type) + innerCaptures
+    case .namedCapture(let name):
+      return .atom(name: name.value, type: type)
+            + innerCaptures
+    default:
+      return innerCaptures
+    }
+  }
+
   // TODO: We'll likely want/need a generalization of
   // conditional's condition kind.
   public mutating func condition<T: _TreeNode>(

Sources/_StringProcessing/ByteCodeGen.swift

@@ -246,12 +246,42 @@ extension Compiler.ByteCodeGen {
     builder.label(success)
   }
 
+  mutating func emitMatcher(
+    _ matcher: @escaping _MatcherInterface,
+    into capture: CaptureRegister? = nil
+  ) {
+
+    // TODO: Consider emitting consumer interface if
+    // not captured. This may mean we should store
+    // an existential instead of a closure...
+
+    let matcher = builder.makeMatcherFunction(matcher)
+
+    let valReg = builder.makeValueRegister()
+    builder.buildMatcher(matcher, into: valReg)
+
+    // TODO: Instruction to store directly
+    if let cap = capture {
+      builder.buildMove(valReg, into: cap)
+    }
+  }
+
   mutating func emitGroup(
     _ kind: AST.Group.Kind, _ child: DSLTree.Node
   ) throws -> CaptureRegister? {
     options.beginScope()
     defer { options.endScope() }
 
+    // If we have a strong type, write result directly into
+    // the capture register.
+    //
+    // FIXME: Unify with .groupTransform
+    if kind.isCapturing, case let .matcher(_, m) = child {
+      let cap = builder.makeCapture()
+      emitMatcher(m, into: cap)
+      return cap
+    }
+
     if let lookaround = kind.lookaroundKind {
       try emitLookaround(lookaround, child)
       return nil
@@ -541,8 +571,10 @@ extension Compiler.ByteCodeGen {
       throw Unsupported("absent function")
     case .consumer:
       throw Unsupported("consumer")
-    case .consumerValidator:
-      throw Unsupported("consumer validator")
+
+    case let .matcher(_, f):
+      emitMatcher(f)
+
     case .characterPredicate:
       throw Unsupported("character predicates")
 

Sources/_StringProcessing/ConsumerInterface.swift

@@ -42,7 +42,7 @@ extension DSLTree.Node {
 
     case .consumer:
       fatalError("FIXME: Is this where we handle them?")
-    case .consumerValidator:
+    case .matcher:
       fatalError("FIXME: Is this where we handle them?")
     case .characterPredicate:
       fatalError("FIXME: Is this where we handle them?")

Sources/_StringProcessing/Engine/InstPayload.swift

@@ -62,6 +62,8 @@ extension Instruction.Payload {
     case packedEltBool(ElementRegister, BoolRegister)
     case packedPosPos(PositionRegister, PositionRegister)
     case packedCapTran(CaptureRegister, TransformRegister)
+    case packedMatchVal(MatcherRegister, ValueRegister)
+    case packedValueCapture(ValueRegister, CaptureRegister)
   }
 }
 
@@ -289,5 +291,23 @@ extension Instruction.Payload {
   ) {
     interpretPair()
   }
+
+  init(value: ValueRegister, capture: CaptureRegister) {
+    self.init(value, capture)
+  }
+  var pairedValueCapture: (
+    ValueRegister, CaptureRegister
+  ) {
+    interpretPair()
+  }
+
+  init(matcher: MatcherRegister, value: ValueRegister) {
+    self.init(matcher, value)
+  }
+  var pairedMatcherValue: (
+    MatcherRegister, ValueRegister
+  ) {
+    interpretPair()
+  }
 }
 

Sources/_StringProcessing/Engine/Instruction.swift

@@ -167,6 +167,8 @@ extension Instruction {
     /// compare against.
     case assertion
 
+    // MARK: Extension points
+
     /// Advance the input position based on the result by calling the consume
     /// function.
     ///
@@ -176,16 +178,28 @@ extension Instruction {
     /// Custom lookaround assertion operation.
     /// Triggers a failure if customFunction returns false.
     ///
-    ///     assert(
-    ///       _ customFunction: (
-    ///         input: Input,
-    ///         currentPos: Position,
-    ///         bounds: Range<Position>) -> Bool
-    ///     )
+    ///     assert(_ customFunction: (
+    ///       input: Input,
+    ///       currentPos: Position,
+    ///       bounds: Range<Position>
+    ///     ) -> Bool)
     ///
     /// Operands: destination bool register, assert hook register
     case assertBy
 
+    /// Custom value-creating consume operation.
+    ///
+    ///     match(
+    ///       _ matchFunction: (
+    ///         input: Input,
+    ///         bounds: Range<Position>
+    ///       ) -> (Position, Any),
+    ///       into: ValueReg
+    ///     )
+    ///
+    ///
+    case matchBy
+
     // MARK: Matching: Save points
 
     /// Add a save point
@@ -246,6 +260,11 @@ extension Instruction {
     ///
     case transformCapture
 
+    /// Save a value into a capture register
+    ///
+    ///     captureValue(_: ValReg, into _: CapReg)
+    case captureValue
+
     /// Match a previously captured value
     ///
     ///     backreference(_:CapReg)
@@ -273,6 +292,8 @@ extension Instruction {
 
     // TODO: Fused assertions. It seems like we often want to
     // branch based on assertion fail or success.
+
+
   }
 }
 

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -22,6 +22,7 @@ extension MEProgram where Input.Element: Hashable {
     var consumeFunctions: [ConsumeFunction] = []
     var assertionFunctions: [AssertionFunction] = []
     var transformFunctions: [TransformFunction] = []
+    var matcherFunctions: [MatcherFunction] = []
 
     // Map tokens to actual addresses
     var addressTokens: [InstructionAddress?] = []
@@ -32,6 +33,7 @@ extension MEProgram where Input.Element: Hashable {
     var nextIntRegister = IntRegister(0)
     var nextPositionRegister = PositionRegister(0)
     var nextCaptureRegister = CaptureRegister(0)
+    var nextValueRegister = ValueRegister(0)
 
     // Special addresses or instructions
     var failAddressToken: AddressToken? = nil
@@ -237,6 +239,22 @@ extension MEProgram.Builder {
       .init(capture: cap, transform: trans)))
   }
 
+  public mutating func buildMatcher(
+    _ fun: MatcherRegister, into reg: ValueRegister
+  ) {
+    instructions.append(.init(
+      .matchBy,
+      .init(matcher: fun, value: reg)))
+  }
+
+  public mutating func buildMove(
+    _ value: ValueRegister, into capture: CaptureRegister
+  ) {
+    instructions.append(.init(
+      .captureValue,
+      .init(value: value, capture: capture)))
+  }
+
   public mutating func buildBackreference(
     _ cap: CaptureRegister
   ) {
@@ -298,9 +316,11 @@ extension MEProgram.Builder {
     regInfo.bools = nextBoolRegister.rawValue
     regInfo.ints = nextIntRegister.rawValue
     regInfo.positions = nextPositionRegister.rawValue
+    regInfo.values = nextValueRegister.rawValue
     regInfo.consumeFunctions = consumeFunctions.count
     regInfo.assertionFunctions = assertionFunctions.count
     regInfo.transformFunctions = transformFunctions.count
+    regInfo.matcherFunctions = matcherFunctions.count
     regInfo.captures = nextCaptureRegister.rawValue
 
     return MEProgram(
@@ -311,6 +331,7 @@ extension MEProgram.Builder {
       staticConsumeFunctions: consumeFunctions,
       staticAssertionFunctions: assertionFunctions,
       staticTransformFunctions: transformFunctions,
+      staticMatcherFunctions: matcherFunctions,
       registerInfo: regInfo,
       captureStructure: captureStructure)
   }
@@ -399,6 +420,10 @@ extension MEProgram.Builder {
     defer { nextPositionRegister.rawValue += 1 }
     return nextPositionRegister
   }
+  public mutating func makeValueRegister() -> ValueRegister {
+    defer { nextValueRegister.rawValue += 1 }
+    return nextValueRegister
+  }
 
   // Allocate and initialize a register
   public mutating func makeIntRegister(
@@ -456,5 +481,11 @@ extension MEProgram.Builder {
     defer { transformFunctions.append(f) }
     return TransformRegister(transformFunctions.count)
   }
+  public mutating func makeMatcherFunction(
+    _ f: @escaping MEProgram.MatcherFunction
+  ) -> MatcherRegister {
+    defer { matcherFunctions.append(f) }
+    return MatcherRegister(matcherFunctions.count)
+  }
 }
 

Sources/_StringProcessing/Engine/MECapture.swift

@@ -60,8 +60,8 @@ extension Processor {
         assert(stack.isEmpty)
         assert(valueStack.isEmpty)
         assert(currentCaptureBegin == nil)
-      } else {
-        assert(!stack.isEmpty || currentCaptureBegin != nil)
+      } else if currentCaptureBegin == nil {
+        assert(!stack.isEmpty || !valueStack.isEmpty)
       }
       if hasValues {
         // FIXME: how?
@@ -109,10 +109,14 @@ extension Processor {
     }
 
     mutating func registerValue(
-      _ value: Any
+      _ value: Any,
+      overwriteInitial: SavePoint? = nil
     ) {
       _invariantCheck()
       defer { _invariantCheck() }
+      if let sp = overwriteInitial {
+        self.startState = sp
+      }
       valueStack.append(value)
     }
 

Sources/_StringProcessing/Engine/MEProgram.swift

@@ -17,6 +17,8 @@ public struct MEProgram<Input: Collection> where Input.Element: Equatable {
     (Input, Input.Index, Range<Input.Index>) -> Bool
   public typealias TransformFunction =
     (Input, Range<Input.Index>) -> Any?
+  public typealias MatcherFunction =
+    (Input, Range<Input.Index>) -> (Input.Index, Any)?
 
   var instructions: InstructionList<Instruction>
 
@@ -26,6 +28,7 @@ public struct MEProgram<Input: Collection> where Input.Element: Equatable {
   var staticConsumeFunctions: [ConsumeFunction]
   var staticAssertionFunctions: [AssertionFunction]
   var staticTransformFunctions: [TransformFunction]
+  var staticMatcherFunctions: [MatcherFunction]
 
   var registerInfo: RegisterInfo
 

Sources/_StringProcessing/Engine/Processor.swift

@@ -358,6 +358,19 @@ extension Processor {
       }
       controller.step()
 
+    case .matchBy:
+      let (matcherReg, valReg) = payload.pairedMatcherValue
+      let matcher = registers[matcherReg]
+      guard let (nextIdx, val) = matcher(
+        input, currentPosition..<bounds.upperBound
+      ) else {
+        signalFailure()
+        return
+      }
+      registers[valReg] = val
+      advance(to: nextIdx)
+      controller.step()
+
     case .print:
       // TODO: Debug stream
       doPrint(registers[payload.string])
@@ -423,6 +436,16 @@ extension Processor {
       storedCaptures[capNum].registerValue(value)
 
       controller.step()
+
+    case .captureValue:
+      let (val, cap) = payload.pairedValueCapture
+      let value = registers[val]
+      let capNum = Int(asserting: cap.rawValue)
+      let sp = makeSavePoint(self.currentPC)
+      storedCaptures[capNum].registerValue(
+        value, overwriteInitial: sp)
+      controller.step()
     }
+
   }
 }